azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[IC] Refactor IC maps to reuse code and ensure consistency - Part 2

Browse Source 
In commit 4e89dcf, we have prepared the API for IC maps in top
interfaces and provide the implementation in abstract classes.

In this commit, we refactor IC maps so that they directly inherit/reuse
the implementation from the superclasses without having to reimplement
the APIs for a map.

Test: Existing tests (refactoring change)
^KT-63456: In progress

Authored-by: Hung Nguyen <hungnv@google.com>

Merge-request: KOTLIN-MR-801
Merged-by: Evgenii Mazhukin <evgenii.mazhukin@jetbrains.com>
This commit is contained in:
Hung Nguyen
2023-11-15 12:22:40 +00:00
committed by Space Cloud
parent c0a05e435c
commit 5562c95155
27 changed files with 473 additions and 578 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt
+19 -29
View File
@@ -16,7 +16,6 @@
package org.jetbrains.kotlin.incremental package org.jetbrains.kotlin.incremental
import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.incremental.storage.*
import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.metadata.ProtoBuf
import org.jetbrains.kotlin.metadata.deserialization.Flags import org.jetbrains.kotlin.metadata.deserialization.Flags
@@ -26,7 +25,6 @@ import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.serialization.deserialization.getClassId import org.jetbrains.kotlin.serialization.deserialization.getClassId
import java.io.File import java.io.File
import java.util.* import java.util.*
import kotlin.collections.HashSet
/** /**
* Incremental cache common for JVM and JS, ClassName type aware * Incremental cache common for JVM and JS, ClassName type aware
@@ -94,13 +92,13 @@ abstract class AbstractIncrementalCache<ClassName>(
private val complementaryFilesMap = registerMap(ComplementarySourceFilesMap(COMPLEMENTARY_FILES.storageFile, icContext)) private val complementaryFilesMap = registerMap(ComplementarySourceFilesMap(COMPLEMENTARY_FILES.storageFile, icContext))
override fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName> = override fun classesFqNamesBySources(files: Iterable<File>): Collection<FqName> =
files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) } files.flatMapTo(mutableSetOf()) { sourceToClassesMap.getFqNames(it).orEmpty() }
override fun getSubtypesOf(className: FqName): Sequence<FqName> = override fun getSubtypesOf(className: FqName): Sequence<FqName> =
subtypesMap[className].asSequence() subtypesMap[className].orEmpty().asSequence()
override fun getSupertypesOf(className: FqName): Sequence<FqName> { override fun getSupertypesOf(className: FqName): Sequence<FqName> {
return supertypesMap[className].asSequence() return supertypesMap[className].orEmpty().asSequence()
} }
override fun isSealed(className: FqName): Boolean? { override fun isSealed(className: FqName): Boolean? {
@@ -112,10 +110,10 @@ abstract class AbstractIncrementalCache<ClassName>(
override fun markDirty(removedAndCompiledSources: Collection<File>) { override fun markDirty(removedAndCompiledSources: Collection<File>) {
for (sourceFile in removedAndCompiledSources) { for (sourceFile in removedAndCompiledSources) {
sourceToClassesMap[sourceFile].forEach { className -> sourceToClassesMap[sourceFile]?.forEach { className ->
markDirty(className) markDirty(className)
} }
sourceToClassesMap.clearOutputsForSource(sourceFile) sourceToClassesMap.remove(sourceFile)
} }
} }
@@ -137,9 +135,9 @@ abstract class AbstractIncrementalCache<ClassName>(
.toSet() .toSet()
val child = nameResolver.getClassId(proto.fqName).asSingleFqName() val child = nameResolver.getClassId(proto.fqName).asSingleFqName()
parents.forEach { subtypesMap.add(it, child) } parents.forEach { subtypesMap.append(it, child) }
val removedSupertypes = supertypesMap[child].filter { it !in parents } val removedSupertypes = supertypesMap[child].orEmpty().filter { it !in parents }
removedSupertypes.forEach { subtypesMap.removeValues(it, setOf(child)) } removedSupertypes.forEach { subtypesMap.removeValues(it, setOf(child)) }
supertypesMap[child] = parents supertypesMap[child] = parents
@@ -163,8 +161,8 @@ abstract class AbstractIncrementalCache<ClassName>(
val childrenFqNames = hashSetOf<FqName>() val childrenFqNames = hashSetOf<FqName>()
for (removedFqName in removedFqNames) { for (removedFqName in removedFqNames) {
parentsFqNames.addAll(cache.supertypesMap[removedFqName]) parentsFqNames.addAll(cache.supertypesMap[removedFqName].orEmpty())
childrenFqNames.addAll(cache.subtypesMap[removedFqName]) childrenFqNames.addAll(cache.subtypesMap[removedFqName].orEmpty())
cache.supertypesMap.remove(removedFqName) cache.supertypesMap.remove(removedFqName)
cache.subtypesMap.remove(removedFqName) cache.subtypesMap.remove(removedFqName)
@@ -188,20 +186,12 @@ abstract class AbstractIncrementalCache<ClassName>(
protected class ClassFqNameToSourceMap( protected class ClassFqNameToSourceMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<String>(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor, icContext) { ) : AbstractBasicMap<FqName, File>(
operator fun set(fqName: FqName, sourceFile: File) { storageFile,
storage[fqName.asString()] = pathConverter.toPath(sourceFile) FqNameExternalizer.toDescriptor(),
} icContext.fileDescriptorForSourceFiles,
icContext
operator fun get(fqName: FqName): File? = )
storage[fqName.asString()]?.let(pathConverter::toFile)
fun remove(fqName: FqName) {
storage.remove(fqName.asString())
}
override fun dumpValue(value: String) = value
}
override fun getComplementaryFilesRecursive(dirtyFiles: Collection<File>): Collection<File> { override fun getComplementaryFilesRecursive(dirtyFiles: Collection<File>): Collection<File> {
val complementaryFiles = HashSet<File>() val complementaryFiles = HashSet<File>()
@@ -216,10 +206,10 @@ abstract class AbstractIncrementalCache<ClassName>(
continue continue
} }
processedFiles.add(file) processedFiles.add(file)
complementaryFilesMap[file].forEach { complementaryFilesMap[file]?.forEach {
if (complementaryFiles.add(it) && !processedFiles.contains(it)) filesQueue.add(it) if (complementaryFiles.add(it) && !processedFiles.contains(it)) filesQueue.add(it)
} }
val classes2recompile = sourceToClassesMap.getFqNames(file) val classes2recompile = sourceToClassesMap.getFqNames(file).orEmpty()
classes2recompile.filter { !processedClasses.contains(it) }.forEach { class2recompile -> classes2recompile.filter { !processedClasses.contains(it) }.forEach { class2recompile ->
processedClasses.add(class2recompile) processedClasses.add(class2recompile)
val sealedClasses = findSealedSupertypes(class2recompile, listOf(this)) val sealedClasses = findSealedSupertypes(class2recompile, listOf(this))
@@ -248,11 +238,11 @@ abstract class AbstractIncrementalCache<ClassName>(
for (actual in actuals) { for (actual in actuals) {
actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect) actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect)
} }
complementaryFilesMap[expect] = actuals.union(complementaryFilesMap[expect]) complementaryFilesMap[expect] = actuals.union(complementaryFilesMap[expect].orEmpty())
} }
for ((actual, expects) in actualToExpect) { for ((actual, expects) in actualToExpect) {
complementaryFilesMap[actual] = expects.union(complementaryFilesMap[actual]) complementaryFilesMap[actual] = expects.union(complementaryFilesMap[actual].orEmpty())
} }
} }
} }
build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt
+8 -7
View File
@@ -5,15 +5,17 @@
package org.jetbrains.kotlin.incremental package org.jetbrains.kotlin.incremental
import com.intellij.util.io.KeyDescriptor
import org.jetbrains.kotlin.build.report.DoNothingICReporter import org.jetbrains.kotlin.build.report.DoNothingICReporter
import org.jetbrains.kotlin.build.report.ICReporter import org.jetbrains.kotlin.build.report.ICReporter
import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter import org.jetbrains.kotlin.incremental.storage.BasicFileToPathConverter
import org.jetbrains.kotlin.incremental.storage.FileToPathConverter import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
import java.io.File
class IncrementalCompilationContext( class IncrementalCompilationContext(
// The root directories of source files and output files are different, so we may need different `FileToPathConverter`s // The root directories of source files and output files are different, so we need different `FileToPathConverter`s
val pathConverterForSourceFiles: FileToPathConverter = FileToAbsolutePathConverter, val pathConverterForSourceFiles: FileToPathConverter = BasicFileToPathConverter,
val pathConverterForOutputFiles: FileToPathConverter = FileToAbsolutePathConverter, val pathConverterForOutputFiles: FileToPathConverter = BasicFileToPathConverter,
val storeFullFqNamesInLookupCache: Boolean = false, val storeFullFqNamesInLookupCache: Boolean = false,
val transaction: CompilationTransaction = NonRecoverableCompilationTransaction(), val transaction: CompilationTransaction = NonRecoverableCompilationTransaction(),
val reporter: ICReporter = DoNothingICReporter, val reporter: ICReporter = DoNothingICReporter,
@@ -46,7 +48,6 @@ class IncrementalCompilationContext(
keepIncrementalCompilationCachesInMemory keepIncrementalCompilationCachesInMemory
) )
// FIXME: Remove `pathConverter` and require its users to decide whether to use `pathConverterForSourceFiles` or val fileDescriptorForSourceFiles: KeyDescriptor<File> = pathConverterForSourceFiles.getFileDescriptor()
// `pathConverterForOutputFiles` val fileDescriptorForOutputFiles: KeyDescriptor<File> = pathConverterForOutputFiles.getFileDescriptor()
val pathConverter = pathConverterForSourceFiles
} }
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt
+40 -44
View File
@@ -17,6 +17,7 @@
package org.jetbrains.kotlin.incremental package org.jetbrains.kotlin.incremental
import com.intellij.util.io.DataExternalizer import com.intellij.util.io.DataExternalizer
import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.build.GeneratedFile
import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl
import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue
@@ -79,7 +80,7 @@ open class IncrementalJsCache(
removedAndCompiledSources.forEach { sourceFile -> removedAndCompiledSources.forEach { sourceFile ->
sourceToJsOutputsMap.remove(sourceFile) sourceToJsOutputsMap.remove(sourceFile)
// The common prefix of all FQN parents has to be the file package // The common prefix of all FQN parents has to be the file package
sourceToClassesMap[sourceFile].map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let { sourceToClassesMap[sourceFile].orEmpty().map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let {
packageMetadata.remove(it) packageMetadata.remove(it)
} }
} }
@@ -99,7 +100,7 @@ open class IncrementalJsCache(
} }
fun getOutputsBySource(sourceFile: File): Collection<File> { fun getOutputsBySource(sourceFile: File): Collection<File> {
return sourceToJsOutputsMap[sourceFile] return sourceToJsOutputsMap[sourceFile].orEmpty()
} }
fun compareAndUpdate(incrementalResults: IncrementalResultsConsumerImpl, changesCollector: ChangesCollector) { fun compareAndUpdate(incrementalResults: IncrementalResultsConsumerImpl, changesCollector: ChangesCollector) {
@@ -132,7 +133,7 @@ open class IncrementalJsCache(
} }
for ((packageName, metadata) in incrementalResults.packageMetadata) { for ((packageName, metadata) in incrementalResults.packageMetadata) {
packageMetadata.put(packageName, metadata) packageMetadata[packageName] = metadata
} }
for ((srcFile, irData) in incrementalResults.irFileData) { for ((srcFile, irData) in incrementalResults.irFileData) {
@@ -142,7 +143,7 @@ open class IncrementalJsCache(
} }
private fun registerOutputForFile(srcFile: File, name: FqName) { private fun registerOutputForFile(srcFile: File, name: FqName) {
sourceToClassesMap.add(srcFile, name) sourceToClassesMap.append(srcFile, name)
dirtyOutputClassesMap.notDirty(name) dirtyOutputClassesMap.notDirty(name)
} }
@@ -159,7 +160,7 @@ open class IncrementalJsCache(
fun nonDirtyPackageParts(): Map<File, TranslationResultValue> = fun nonDirtyPackageParts(): Map<File, TranslationResultValue> =
hashMapOf<File, TranslationResultValue>().apply { hashMapOf<File, TranslationResultValue>().apply {
for (file in translationResults.keys()) { for (file in translationResults.keys) {
if (file !in dirtySources) { if (file !in dirtySources) {
put(file, translationResults[file]!!) put(file, translationResults[file]!!)
@@ -175,7 +176,7 @@ open class IncrementalJsCache(
fun nonDirtyIrParts(): Map<File, IrTranslationResultValue> = fun nonDirtyIrParts(): Map<File, IrTranslationResultValue> =
hashMapOf<File, IrTranslationResultValue>().apply { hashMapOf<File, IrTranslationResultValue>().apply {
for (file in irTranslationResults.keys()) { for (file in irTranslationResults.keys) {
if (file !in dirtySources) { if (file !in dirtySources) {
put(file, irTranslationResults[file]!!) put(file, irTranslationResults[file]!!)
@@ -189,7 +190,7 @@ open class IncrementalJsCache(
for (generatedFile in generatedFiles) { for (generatedFile in generatedFiles) {
for (source in generatedFile.sourceFiles) { for (source in generatedFile.sourceFiles) {
if (dirtySources.contains(source)) if (dirtySources.contains(source))
sourceToJsOutputsMap.add(source, generatedFile.outputFile) sourceToJsOutputsMap.append(source, generatedFile.outputFile)
} }
} }
} }
@@ -228,34 +229,32 @@ private class TranslationResultMap(
storageFile: File, storageFile: File,
private val protoData: ProtoDataProvider, private val protoData: ProtoDataProvider,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ) : AbstractBasicMap<File, TranslationResultValue>(
BasicStringMap<TranslationResultValue>(storageFile, TranslationResultValueExternalizer, icContext) { storageFile,
icContext.fileDescriptorForSourceFiles,
TranslationResultValueExternalizer,
icContext
) {
@TestOnly
override fun dumpValue(value: TranslationResultValue): String = override fun dumpValue(value: TranslationResultValue): String =
"Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}, InlineData: ${value.inlineData.md5()}" "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}, InlineData: ${value.inlineData.md5()}"
@Synchronized @Synchronized
fun put(sourceFile: File, newMetadata: ByteArray, newBinaryAst: ByteArray, newInlineData: ByteArray) { fun put(sourceFile: File, newMetadata: ByteArray, newBinaryAst: ByteArray, newInlineData: ByteArray) {
storage[pathConverter.toPath(sourceFile)] = this[sourceFile] =
TranslationResultValue(metadata = newMetadata, binaryAst = newBinaryAst, inlineData = newInlineData) TranslationResultValue(metadata = newMetadata, binaryAst = newBinaryAst, inlineData = newInlineData)
} }
@Synchronized
operator fun get(sourceFile: File): TranslationResultValue? =
storage[pathConverter.toPath(sourceFile)]
fun keys(): Collection<File> =
storage.keys.map { pathConverter.toFile(it) }
@Synchronized @Synchronized
fun remove(sourceFile: File, changesCollector: ChangesCollector) { fun remove(sourceFile: File, changesCollector: ChangesCollector) {
val path = pathConverter.toPath(sourceFile) val protoBytes = this[sourceFile]!!.metadata
val protoBytes = storage[path]!!.metadata
val protoMap = protoData(sourceFile, protoBytes) val protoMap = protoData(sourceFile, protoBytes)
for ((_, protoData) in protoMap) { for ((_, protoData) in protoMap) {
changesCollector.collectProtoChanges(oldData = protoData, newData = null) changesCollector.collectProtoChanges(oldData = protoData, newData = null)
} }
storage.remove(path) remove(sourceFile)
} }
} }
@@ -313,8 +312,14 @@ private object IrTranslationResultValueExternalizer : DataExternalizer<IrTransla
private class IrTranslationResultMap( private class IrTranslationResultMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ) : AbstractBasicMap<File, IrTranslationResultValue>(
BasicStringMap<IrTranslationResultValue>(storageFile, IrTranslationResultValueExternalizer, icContext) { storageFile,
icContext.fileDescriptorForSourceFiles,
IrTranslationResultValueExternalizer,
icContext
) {
@TestOnly
override fun dumpValue(value: IrTranslationResultValue): String = override fun dumpValue(value: IrTranslationResultValue): String =
"Filedata: ${value.fileData.md5()}, " + "Filedata: ${value.fileData.md5()}, " +
"Types: ${value.types.md5()}, " + "Types: ${value.types.md5()}, " +
@@ -323,6 +328,7 @@ private class IrTranslationResultMap(
"Declarations: ${value.declarations.md5()}, " + "Declarations: ${value.declarations.md5()}, " +
"Bodies: ${value.bodies.md5()}" "Bodies: ${value.bodies.md5()}"
@Synchronized
fun put( fun put(
sourceFile: File, sourceFile: File,
newFiledata: ByteArray, newFiledata: ByteArray,
@@ -335,20 +341,9 @@ private class IrTranslationResultMap(
newFileMetadata: ByteArray, newFileMetadata: ByteArray,
debugInfos: ByteArray?, debugInfos: ByteArray?,
) { ) {
storage[pathConverter.toPath(sourceFile)] = this[sourceFile] =
IrTranslationResultValue(newFiledata, newTypes, newSignatures, newStrings, newDeclarations, newBodies, fqn, newFileMetadata, debugInfos) IrTranslationResultValue(newFiledata, newTypes, newSignatures, newStrings, newDeclarations, newBodies, fqn, newFileMetadata, debugInfos)
} }
operator fun get(sourceFile: File): IrTranslationResultValue? =
storage[pathConverter.toPath(sourceFile)]
fun keys(): Collection<File> =
storage.keys.map { pathConverter.toFile(it) }
fun remove(sourceFile: File) {
val path = pathConverter.toPath(sourceFile)
storage.remove(path)
}
} }
private class ProtoDataProvider(private val serializerProtocol: SerializerExtensionProtocol) { private class ProtoDataProvider(private val serializerProtocol: SerializerExtensionProtocol) {
@@ -396,16 +391,21 @@ fun getProtoData(sourceFile: File, metadata: ByteArray): Map<ClassId, ProtoData>
private class InlineFunctionsMap( private class InlineFunctionsMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<Map<String, Long>>(storageFile, StringToLongMapExternalizer, icContext) { ) : AbstractBasicMap<File, Map<String, Long>>(
storageFile,
icContext.fileDescriptorForSourceFiles,
StringToLongMapExternalizer,
icContext
) {
@Synchronized @Synchronized
fun process(srcFile: File, newMap: Map<String, Long>, changesCollector: ChangesCollector) { fun process(srcFile: File, newMap: Map<String, Long>, changesCollector: ChangesCollector) {
val key = pathConverter.toPath(srcFile) val oldMap = this[srcFile] ?: emptyMap()
val oldMap = storage[key] ?: emptyMap()
if (newMap.isNotEmpty()) { if (newMap.isNotEmpty()) {
storage[key] = newMap this[srcFile] = newMap
} else { } else {
storage.remove(key) remove(srcFile)
} }
for (fn in oldMap.keys + newMap.keys) { for (fn in oldMap.keys + newMap.keys) {
@@ -415,11 +415,7 @@ private class InlineFunctionsMap(
} }
} }
@Synchronized @TestOnly
fun remove(sourceFile: File) {
storage.remove(pathConverter.toPath(sourceFile))
}
override fun dumpValue(value: Map<String, Long>): String = override fun dumpValue(value: Map<String, Long>): String =
value.dumpMap { java.lang.Long.toHexString(it) } value.dumpMap { java.lang.Long.toHexString(it) }
} }
build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt
+26 -62
View File
@@ -17,10 +17,7 @@
package org.jetbrains.kotlin.incremental package org.jetbrains.kotlin.incremental
import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
import com.intellij.util.containers.CollectionFactory
import com.intellij.util.io.BooleanDataDescriptor import com.intellij.util.io.BooleanDataDescriptor
import com.intellij.util.io.EnumeratorStringDescriptor
import gnu.trove.THashSet
import org.jetbrains.annotations.TestOnly import org.jetbrains.annotations.TestOnly
import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.build.GeneratedJvmClass
import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.incremental.storage.*
@@ -88,13 +85,10 @@ open class IncrementalJvmCache(
// used in gradle // used in gradle
@Suppress("unused") @Suppress("unused")
fun classesBySources(sources: Iterable<File>): Iterable<JvmClassName> = fun classesBySources(sources: Iterable<File>): Iterable<JvmClassName> =
sources.flatMap { sourceToClassesMap[it] } sources.flatMap { sourceToClassesMap[it].orEmpty() }
fun sourceInCache(file: File): Boolean =
sourceToClassesMap.contains(file)
fun sourcesByInternalName(internalName: String): Collection<File> = fun sourcesByInternalName(internalName: String): Collection<File> =
internalNameToSource.getFiles(internalName) internalNameToSource[internalName].orEmpty()
fun getAllPartsOfMultifileFacade(facade: JvmClassName): Collection<String>? { fun getAllPartsOfMultifileFacade(facade: JvmClassName): Collection<String>? {
return multifileFacadeToParts[facade] return multifileFacadeToParts[facade]
@@ -111,7 +105,7 @@ open class IncrementalJvmCache(
val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME) val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME)
protoMap.storeModuleMapping(jvmClassName, file.readBytes()) protoMap.storeModuleMapping(jvmClassName, file.readBytes())
dirtyOutputClassesMap.notDirty(jvmClassName) dirtyOutputClassesMap.notDirty(jvmClassName)
sourceFiles.forEach { sourceToClassesMap.add(it, jvmClassName) } sourceFiles.forEach { sourceToClassesMap.append(it, jvmClassName) }
} }
open fun saveFileToCache(generatedClass: GeneratedJvmClass, changesCollector: ChangesCollector) { open fun saveFileToCache(generatedClass: GeneratedJvmClass, changesCollector: ChangesCollector) {
@@ -133,7 +127,7 @@ open class IncrementalJvmCache(
if (sourceFiles != null) { if (sourceFiles != null) {
sourceFiles.forEach { sourceFiles.forEach {
sourceToClassesMap.add(it, className) sourceToClassesMap.append(it, className)
} }
internalNameToSource[className.internalName] = sourceFiles internalNameToSource[className.internalName] = sourceFiles
} }
@@ -176,7 +170,7 @@ open class IncrementalJvmCache(
assert(sourceFiles.size == 1) { "Multifile class part from several source files: $sourceFiles" } assert(sourceFiles.size == 1) { "Multifile class part from several source files: $sourceFiles" }
} }
packagePartMap.addPackagePart(className) packagePartMap.addPackagePart(className)
partToMultifileFacade.set(className.internalName, kotlinClassInfo.multifileClassName!!) partToMultifileFacade[className] = kotlinClassInfo.multifileClassName!!
protoMap.process(kotlinClassInfo, changesCollector) protoMap.process(kotlinClassInfo, changesCollector)
constantsMap.process(kotlinClassInfo, changesCollector) constantsMap.process(kotlinClassInfo, changesCollector)
@@ -235,7 +229,7 @@ open class IncrementalJvmCache(
fun saveJavaClassProto(source: File?, serializedJavaClass: SerializedJavaClass, collector: ChangesCollector) { fun saveJavaClassProto(source: File?, serializedJavaClass: SerializedJavaClass, collector: ChangesCollector) {
val jvmClassName = JvmClassName.byClassId(serializedJavaClass.classId) val jvmClassName = JvmClassName.byClassId(serializedJavaClass.classId)
javaSourcesProtoMap.process(jvmClassName, serializedJavaClass, collector) javaSourcesProtoMap.process(jvmClassName, serializedJavaClass, collector)
source?.let { sourceToClassesMap.add(source, jvmClassName) } source?.let { sourceToClassesMap.append(source, jvmClassName) }
addToClassStorage(serializedJavaClass.toProtoData(), source) addToClassStorage(serializedJavaClass.toProtoData(), source)
// collector.addJavaProto(ClassProtoData(proto, nameResolver)) // collector.addJavaProto(ClassProtoData(proto, nameResolver))
dirtyOutputClassesMap.notDirty(jvmClassName) dirtyOutputClassesMap.notDirty(jvmClassName)
@@ -263,7 +257,7 @@ open class IncrementalJvmCache(
val facadesWithRemovedParts = hashMapOf<JvmClassName, MutableSet<String>>() val facadesWithRemovedParts = hashMapOf<JvmClassName, MutableSet<String>>()
for (dirtyClass in dirtyClasses) { for (dirtyClass in dirtyClasses) {
val facade = partToMultifileFacade.get(dirtyClass) ?: continue val facade = partToMultifileFacade[dirtyClass] ?: continue
val facadeClassName = JvmClassName.byInternalName(facade) val facadeClassName = JvmClassName.byInternalName(facade)
val removedParts = facadesWithRemovedParts.getOrPut(facadeClassName) { hashSetOf() } val removedParts = facadesWithRemovedParts.getOrPut(facadeClassName) { hashSetOf() }
removedParts.add(dirtyClass.internalName) removedParts.add(dirtyClass.internalName)
@@ -311,7 +305,7 @@ open class IncrementalJvmCache(
override fun getObsoleteMultifileClasses(): Collection<String> { override fun getObsoleteMultifileClasses(): Collection<String> {
val obsoleteMultifileClasses = linkedSetOf<String>() val obsoleteMultifileClasses = linkedSetOf<String>()
for (dirtyClass in dirtyOutputClassesMap.getDirtyOutputClasses()) { for (dirtyClass in dirtyOutputClassesMap.getDirtyOutputClasses()) {
val dirtyFacade = partToMultifileFacade.get(dirtyClass) ?: continue val dirtyFacade = partToMultifileFacade[dirtyClass] ?: continue
obsoleteMultifileClasses.add(dirtyFacade) obsoleteMultifileClasses.add(dirtyFacade)
} }
debugLog("Obsolete multifile class facades: $obsoleteMultifileClasses") debugLog("Obsolete multifile class facades: $obsoleteMultifileClasses")
@@ -514,59 +508,32 @@ open class IncrementalJvmCache(
private inner class MultifileClassFacadeMap( private inner class MultifileClassFacadeMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ) : AppendableBasicMap<JvmClassName, String>(
BasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer, icContext) { storageFile,
JvmClassNameExternalizer.toDescriptor(),
@Synchronized StringExternalizer,
operator fun set(className: JvmClassName, partNames: Collection<String>) { icContext
storage[className.internalName] = partNames )
}
operator fun get(className: JvmClassName): Collection<String>? =
storage[className.internalName]
operator fun contains(className: JvmClassName): Boolean =
className.internalName in storage
@Synchronized
fun remove(className: JvmClassName) {
storage.remove(className.internalName)
}
override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
}
private inner class MultifileClassPartMap( private inner class MultifileClassPartMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ) : AbstractBasicMap<JvmClassName, String>(
BasicStringMap<String>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { storageFile,
JvmClassNameExternalizer.toDescriptor(),
fun get(partName: JvmClassName): String? = StringExternalizer,
storage[partName.internalName] icContext
)
@Synchronized
fun remove(className: JvmClassName) {
storage.remove(className.internalName)
}
override fun dumpValue(value: String): String = value
}
inner class InternalNameToSourcesMap( inner class InternalNameToSourcesMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<Collection<String>>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer, icContext) { ) : AppendableBasicMap<String, File>(
operator fun set(internalName: String, sourceFiles: Collection<File>) { storageFile,
storage[internalName] = pathConverter.toPaths(sourceFiles) StringExternalizer.toDescriptor(),
} icContext.fileDescriptorForSourceFiles,
icContext
fun getFiles(internalName: String): Collection<File> = )
pathConverter.toFiles(storage[internalName] ?: emptyList())
override fun dumpValue(value: Collection<String>): String =
value.dumpCollection()
}
private inner class InlineFunctionsMap( private inner class InlineFunctionsMap(
storageFile: File, storageFile: File,
@@ -628,9 +595,6 @@ open class IncrementalJvmCache(
} }
} }
private object PathCollectionExternalizer :
CollectionExternalizerV2<String, Collection<String>>(PathStringDescriptor, { THashSet(CollectionFactory.createFilePathSet()) })
sealed class ChangeInfo(val fqName: FqName) { sealed class ChangeInfo(val fqName: FqName) {
open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName) { open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName) {
override fun toStringProperties(): String = super.toStringProperties() + ", names = $names" override fun toStringProperties(): String = super.toStringProperties() + ", names = $names"
build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt
+13 -17
View File
@@ -96,7 +96,7 @@ open class LookupStorage(
val filtered = mutableSetOf<Int>() val filtered = mutableSetOf<Int>()
for (fileId in fileIds) { for (fileId in fileIds) {
val path = idToFile.getFile(fileId)?.path val path = idToFile[fileId]?.path
if (path != null) { if (path != null) {
paths.add(path) paths.add(path)
@@ -171,7 +171,7 @@ open class LookupStorage(
lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet() lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet()
} }
val oldFileToId = fileToId.toMap() val oldFileToId = fileToId.keys.associateWith { fileToId[it]!! }
val oldIdToNewId = HashMap<Int, Int>(oldFileToId.size) val oldIdToNewId = HashMap<Int, Int>(oldFileToId.size)
idToFile.clear() idToFile.clear()
fileToId.clear() fileToId.clear()
@@ -302,17 +302,17 @@ private class TrackedLookupMap(private val lookupMap: LookupMap, private val tra
val addedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null val addedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
val removedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null val removedKeys = if (trackChanges) mutableSetOf<LookupSymbolKey>() else null
val keys: Collection<LookupSymbolKey> val keys: Set<LookupSymbolKey>
get() = lookupMap.keys get() = lookupMap.keys
operator fun get(key: LookupSymbolKey): Collection<Int>? = lookupMap[key] operator fun get(key: LookupSymbolKey): Set<Int>? = lookupMap[key]
operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) { operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {
recordSet(key) recordSet(key)
lookupMap[key] = fileIds lookupMap[key] = fileIds
} }
fun append(key: LookupSymbolKey, fileIds: Collection<Int>) { fun append(key: LookupSymbolKey, fileIds: Set<Int>) {
recordSet(key) recordSet(key)
lookupMap.append(key, fileIds) lookupMap.append(key, fileIds)
} }
@@ -324,23 +324,19 @@ private class TrackedLookupMap(private val lookupMap: LookupMap, private val tra
private fun recordSet(key: LookupSymbolKey) { private fun recordSet(key: LookupSymbolKey) {
if (!trackChanges) return if (!trackChanges) return
if (lookupMap[key] == null) { when (key) {
if (key in removedKeys!!) { in lookupMap -> Unit
removedKeys.remove(key) in removedKeys!! -> removedKeys.remove(key)
} else { else -> addedKeys!!.add(key)
addedKeys!!.add(key)
}
} }
} }
private fun recordRemove(key: LookupSymbolKey) { private fun recordRemove(key: LookupSymbolKey) {
if (!trackChanges) return if (!trackChanges) return
if (lookupMap[key] != null) { when (key) {
if (key in addedKeys!!) { !in lookupMap -> Unit
addedKeys.remove(key) in addedKeys!! -> addedKeys.remove(key)
} else { else -> removedKeys!!.add(key)
removedKeys!!.add(key)
}
} }
} }
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt
+103 -41
View File
@@ -17,7 +17,6 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.DataExternalizer import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.IOUtil import com.intellij.util.io.IOUtil
import com.intellij.util.io.KeyDescriptor import com.intellij.util.io.KeyDescriptor
import org.jetbrains.annotations.TestOnly import org.jetbrains.annotations.TestOnly
@@ -30,7 +29,9 @@ interface BasicMap<KEY, VALUE> : PersistentStorage<KEY, VALUE> {
/** Removes all entries. */ /** Removes all entries. */
fun clear() { fun clear() {
keys.forEach { remove(it) } synchronized(this) {
keys.forEach { remove(it) }
}
} }
/** /**
@@ -40,22 +41,29 @@ interface BasicMap<KEY, VALUE> : PersistentStorage<KEY, VALUE> {
* Make sure the storage has been closed first before calling this method. * Make sure the storage has been closed first before calling this method.
*/ */
fun deleteStorageFiles() { fun deleteStorageFiles() {
check(IOUtil.deleteAllFilesStartingWith(storageFile)) { synchronized(this) {
"Unable to delete storage file(s) with name prefix: ${storageFile.path}" check(IOUtil.deleteAllFilesStartingWith(storageFile)) {
"Unable to delete storage file(s) with name prefix: ${storageFile.path}"
}
} }
} }
@TestOnly @TestOnly
fun dump(): String { fun dump(): String {
val map: Map<KEY, VALUE> = synchronized(this) {
keys.associateWith { this[it]!! }
}
val printableMap: Map<String, String> = map.map {
dumpKey(it.key) to dumpValue(it.value)
}.sortedBy { it.first }.toMap()
return StringBuilder().apply { return StringBuilder().apply {
Printer(this).apply { Printer(this).apply {
println("${storageFile.name.substringBefore(".tab")} (${this@BasicMap::class.java.simpleName})") println("${storageFile.name.substringBefore(".tab")} (${this@BasicMap::class.java.simpleName})")
pushIndent() pushIndent()
printableMap.forEach {
for (key in keys.sortedBy { dumpKey(it) }) { println("${it.key} -> ${it.value}")
println("${dumpKey(key)} -> ${dumpValue(this@BasicMap[key]!!)}")
} }
popIndent() popIndent()
} }
}.toString() }.toString()
@@ -65,59 +73,113 @@ interface BasicMap<KEY, VALUE> : PersistentStorage<KEY, VALUE> {
fun dumpKey(key: KEY): String = key.toString() fun dumpKey(key: KEY): String = key.toString()
@TestOnly @TestOnly
fun dumpValue(value: VALUE): String = value.toString() fun dumpValue(value: VALUE): String {
return if (value is Collection<*>) {
// Sort the elements so that we can reliably compare `Collection`s in tests (in case the order of the elements is not stable).
value.sortedBy { it.toString() }.toString()
} else {
value.toString()
}
}
} }
abstract class BasicMapBase<KEY, VALUE>( abstract class BasicMapBase<KEY, VALUE>(
protected val storage: PersistentStorage<KEY, VALUE>, // TODO(KT-63456): Remove `protected val` (currently this property is still used by BasicStringMap)
) : PersistentStorageWrapper<KEY, VALUE>(storage), BasicMap<KEY, VALUE> protected val persistentStorage: PersistentStorage<KEY, VALUE>,
) : PersistentStorageWrapper<KEY, VALUE>(persistentStorage), BasicMap<KEY, VALUE>
abstract class AppendableBasicMapBase<KEY, E, VALUE : Collection<E>>( abstract class AppendableBasicMapBase<KEY, E>(
protected val storage: AppendablePersistentStorage<KEY, E, VALUE>, storage: AppendablePersistentStorage<KEY, E>,
) : AppendablePersistentStorageWrapper<KEY, E, VALUE>(storage), BasicMap<KEY, VALUE> ) : AppendablePersistentStorageWrapper<KEY, E>(storage), BasicMap<KEY, Collection<E>>
abstract class AbstractBasicMap<KEY, VALUE>( abstract class AbstractBasicMap<KEY, VALUE>(
storageFile: File, storageFile: File,
keyDescriptor: KeyDescriptor<KEY>, keyDescriptor: KeyDescriptor<KEY>,
valueExternalizer: DataExternalizer<VALUE>, valueExternalizer: DataExternalizer<VALUE>,
protected val icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicMapBase<KEY, VALUE>( ) : BasicMapBase<KEY, VALUE>(
createPersistentStorage(storageFile, keyDescriptor, valueExternalizer, icContext) createPersistentStorage(storageFile, keyDescriptor, valueExternalizer, icContext)
) { )
protected val pathConverter
get() = icContext.pathConverter
}
abstract class AppendableAbstractBasicMap<KEY, E, VALUE : Collection<E>>( abstract class AppendableBasicMap<KEY, E>(
storageFile: File, storageFile: File,
keyDescriptor: KeyDescriptor<KEY>, keyDescriptor: KeyDescriptor<KEY>,
elementExternalizer: DataExternalizer<E>, elementExternalizer: DataExternalizer<E>,
protected val icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AppendableBasicMapBase<KEY, E, VALUE>( ) : AppendableBasicMapBase<KEY, E>(
createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext) createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext)
) { )
protected val pathConverter
get() = icContext.pathConverter abstract class AppendableSetBasicMap<KEY, E>(
storageFile: File,
keyDescriptor: KeyDescriptor<KEY>,
elementExternalizer: DataExternalizer<E>,
icContext: IncrementalCompilationContext,
) : PersistentStorage<KEY, Set<E>>, BasicMap<KEY, Set<E>> {
private val storage = createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext)
override val storageFile: File = storage.storageFile
@get:Synchronized
override val keys: Set<KEY>
get() = storage.keys
@Synchronized
override fun contains(key: KEY): Boolean =
storage.contains(key)
@Synchronized
override fun get(key: KEY): Set<E>? {
// Note: To optimize this getter, consider changing the type of `storage` from PersistentStorage<KEY, Collection<E>> to
// PersistentStorage<KEY, Set<E>> so that we don't have to call `toSet()`. The downside is that it will make the code more complex,
// so we'll do it only if it's necessary.
return storage[key]?.toSet()
}
@Synchronized
override fun set(key: KEY, value: Set<E>) {
storage[key] = value
}
@Synchronized
override fun remove(key: KEY) {
storage.remove(key)
}
@Synchronized
override fun flush() {
storage.flush()
}
@Synchronized
override fun close() {
storage.close()
}
@Synchronized
fun append(key: KEY, element: E) {
storage.append(key, setOf(element))
}
@Synchronized
fun append(key: KEY, elements: Set<E>) {
storage.append(key, elements)
}
} }
abstract class BasicStringMap<VALUE>( abstract class BasicStringMap<VALUE>(
storageFile: File, storageFile: File,
keyDescriptor: KeyDescriptor<String>,
valueExternalizer: DataExternalizer<VALUE>, valueExternalizer: DataExternalizer<VALUE>,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AbstractBasicMap<String, VALUE>(storageFile, keyDescriptor, valueExternalizer, icContext) { ) : AbstractBasicMap<String, VALUE>(
storageFile,
constructor(storageFile: File, valueExternalizer: DataExternalizer<VALUE>, icContext: IncrementalCompilationContext) : StringExternalizer.toDescriptor(),
this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext) valueExternalizer,
} icContext
) {
abstract class AppendableBasicStringMap<E, VALUE : Collection<E>>( // TODO(KT-63456): Remove this property
storageFile: File, // (To do this, we need to refactor all subclasses of BasicStringMap such that they don't use this property. For examples of what the
keyDescriptor: KeyDescriptor<String>, // outcome of the refactoring should look like, see other subclasses of AbstractBasicMap.)
elementExternalizer: DataExternalizer<E>, protected val storage: PersistentStorage<String, VALUE> = persistentStorage
icContext: IncrementalCompilationContext,
) : AppendableAbstractBasicMap<String, E, VALUE>(storageFile, keyDescriptor, elementExternalizer, icContext) {
constructor(storageFile: File, elementExternalizer: DataExternalizer<E>, icContext: IncrementalCompilationContext) :
this(storageFile, EnumeratorStringDescriptor.INSTANCE, elementExternalizer, icContext)
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt
+34 -26
View File
@@ -16,49 +16,38 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import java.io.DataInput
import java.io.DataOutput
import java.io.File import java.io.File
internal open class ClassOneToManyMap( internal open class ClassOneToManyMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AppendableBasicStringMap<String, Collection<String>>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { ) : AppendableSetBasicMap<FqName, FqName>(
override fun dumpValue(value: Collection<String>): String = value.toSet().dumpCollection() storageFile,
LegacyFqNameExternalizer.toDescriptor(),
LegacyFqNameExternalizer,
icContext
) {
@Synchronized @Synchronized
fun add(key: FqName, value: FqName) { override operator fun set(key: FqName, value: Set<FqName>) {
storage.append(key.asString(), value.asString()) if (value.isNotEmpty()) {
} super.set(key, value)
} else {
@Synchronized
operator fun get(key: FqName): Collection<FqName> =
storage[key.asString()]?.mapTo(mutableSetOf(), ::FqName) ?: setOf()
@Synchronized
operator fun set(key: FqName, values: Collection<FqName>) {
if (values.isEmpty()) {
remove(key) remove(key)
return
} }
storage[key.asString()] = values.map(FqName::asString)
} }
@Synchronized
fun remove(key: FqName) {
storage.remove(key.asString())
}
// Access to caches could be done from multiple threads (e.g. JPS worker and RMI). The underlying collection is already synchronized,
// thus we need synchronization of this method and all modification methods.
@Synchronized @Synchronized
fun removeValues(key: FqName, removed: Set<FqName>) { fun removeValues(key: FqName, removed: Set<FqName>) {
val notRemoved = this[key].filter { it !in removed } this[key] = this[key].orEmpty() - removed
this[key] = notRemoved
} }
} }
internal class SubtypesMap( internal class SubtypesMap(
@@ -70,3 +59,22 @@ internal class SupertypesMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ClassOneToManyMap(storageFile, icContext) ) : ClassOneToManyMap(storageFile, icContext)
/**
* Use [LegacyFqNameExternalizer] instead of [FqNameExternalizer] for [SubtypesMap] for now because they internally use different types of
* `DataExternalizer<String>`, and the `compiler-reference-index` module in the Kotlin IDEA plugin currently can only read the old data
* format (see KTIJ-27258).
*
* Once we fix that bug, we can remove this class and use [FqNameExternalizer].
*/
private object LegacyFqNameExternalizer : DataExternalizer<FqName> {
override fun save(output: DataOutput, fqName: FqName) {
EnumeratorStringDescriptor.INSTANCE.save(output, fqName.asString())
}
override fun read(input: DataInput): FqName {
return FqName(EnumeratorStringDescriptor.INSTANCE.read(input))
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt
+4 -22
View File
@@ -5,33 +5,15 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File import java.io.File
class ComplementarySourceFilesMap( class ComplementarySourceFilesMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AppendableBasicStringMap<String, Collection<String>>( ) : AppendableSetBasicMap<File, File>(
storageFile, storageFile,
PathStringDescriptor, icContext.fileDescriptorForSourceFiles,
EnumeratorStringDescriptor.INSTANCE, icContext.fileDescriptorForSourceFiles,
icContext icContext
) { )
operator fun set(sourceFile: File, complementaryFiles: Collection<File>) {
storage[pathConverter.toPath(sourceFile)] = pathConverter.toPaths(complementaryFiles)
}
operator fun get(sourceFile: File): Collection<File> {
val paths = storage[pathConverter.toPath(sourceFile)].orEmpty()
return pathConverter.toFiles(paths).toSet()
}
override fun dumpValue(value: Collection<String>) =
value.dumpCollection()
fun remove(file: File): Collection<File> =
get(file).also { storage.remove(pathConverter.toPath(file)) }
}
build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt
+7 -23
View File
@@ -20,27 +20,11 @@ import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import java.io.File import java.io.File
internal class FileToIdMap( internal class FileToIdMap(
file: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<Int>(file, IntExternalizer, icContext) { ) : AbstractBasicMap<File, Int>(
override fun dumpValue(value: Int): String = value.toString() storageFile,
icContext.fileDescriptorForSourceFiles,
operator fun get(file: File): Int? = storage[pathConverter.toPath(file)] IntExternalizer,
icContext
operator fun set(file: File, id: Int) { )
storage[pathConverter.toPath(file)] = id
}
fun remove(file: File) {
storage.remove(pathConverter.toPath(file))
}
fun toMap(): Map<File, Int> {
val result = HashMap<File, Int>()
for (key in storage.keys) {
val value = storage[key] ?: continue
result[pathConverter.toFile(key)] = value
}
return result
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt
+42 -10
View File
@@ -5,22 +5,54 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.KeyDescriptor
import java.io.DataInput
import java.io.DataOutput
import java.io.File import java.io.File
/** Converts a [File] to a path of type [String] to store in IC caches, and vice versa. */ /** Converts a [File] to a path of type [String] (and vice versa) for serialization of file paths in IC caches. */
interface FileToPathConverter { interface FileToPathConverter {
fun toPath(file: File): String fun toPath(file: File): String
fun toFile(path: String): File fun toFile(path: String): File
fun getFileDescriptor(): KeyDescriptor<File> = FileDescriptor(this)
} }
fun FileToPathConverter.toPaths(files: Collection<File>): List<String> = object BasicFileToPathConverter : FileToPathConverter {
files.map { toPath(it) } override fun toPath(file: File): String = file.path
fun FileToPathConverter.toFiles(paths: Collection<String>): List<File> =
paths.map { toFile(it) }
object FileToAbsolutePathConverter : FileToPathConverter {
override fun toPath(file: File): String = file.normalize().absolutePath
override fun toFile(path: String): File = File(path) override fun toFile(path: String): File = File(path)
} }
/**
* [KeyDescriptor] for a [File] which uses the given [pathConverter] for serialization of file paths.
*
* To support build cache relocatability, the user of this class can provide a [RelocatableFileToPathConverter] as the [pathConverter].
*/
private class FileDescriptor(private val pathConverter: FileToPathConverter) : KeyDescriptor<File> {
override fun save(output: DataOutput, file: File) {
StringExternalizer.save(output, pathConverter.toPath(file))
}
override fun read(input: DataInput): File {
return pathConverter.toFile(StringExternalizer.read(input))
}
override fun getHashCode(file: File): Int {
// It's important to use the pathConverter to get a possibly relocatable path first before computing the hash code because the
// returned hash code affects the contents of IC caches.
// For example, if we get the hash code directly without using pathConverter, the hash codes of the following files will be
// different and therefore the contents of the IC caches will also be different (i.e., the IC caches will not be relocatable):
// File("/path/to/project1/src/foo.kt").hashCode() = 123
// File("/path/to/project2/src/foo.kt").hashCode() = 456
// If we use a RelocatableFileToPathConverter instead, the file paths will be normalized first, so we'll get the same hash codes:
// [In "/path/to/project1"] File("src/foo.kt").hashCode() = 789
// [In "/path/to/project2"] File("src/foo.kt").hashCode() = 789
// It's also important to get the hash code using `File.hashCode()` instead of `String.hashCode()` because the hashCode() method
// needs to account for case-insensitive filesystems.
return File(pathConverter.toPath(file)).hashCode()
}
override fun isEqual(file1: File, file2: File): Boolean {
return file1 == file2
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt
+26 -7
View File
@@ -16,23 +16,42 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.ExternalIntegerKeyDescriptor import com.intellij.util.io.ExternalIntegerKeyDescriptor
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import java.io.DataInput
import java.io.DataOutput
import java.io.File import java.io.File
internal class IdToFileMap( internal class IdToFileMap(
file: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AbstractBasicMap<Int, String>(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE, icContext) { ) : AbstractBasicMap<Int, File>(
override fun dumpKey(key: Int): String = key.toString() storageFile,
LegacyIntExternalizer.toDescriptor(),
LegacyFileExternalizer(icContext.pathConverterForSourceFiles),
icContext
)
override fun dumpValue(value: String): String = value /** Similar to [LegacyFileExternalizer]. */
private val LegacyIntExternalizer = ExternalIntegerKeyDescriptor.INSTANCE
fun getFile(id: Int): File? = storage[id]?.let { pathConverter.toFile(it) } /**
* Use [LegacyFqNameExternalizer] instead of [org.jetbrains.kotlin.incremental.IncrementalCompilationContext.fileDescriptorForOutputFiles]
* for [IdToFileMap] for now because they internally use different types of `DataExternalizer<String>`, and the `compiler-reference-index`
* module in the Kotlin IDEA plugin currently can only read the old data format (see KTIJ-27258).
*
* Once we fix that bug, we can remove this class and use
* [org.jetbrains.kotlin.incremental.IncrementalCompilationContext.fileDescriptorForOutputFiles].
*/
private class LegacyFileExternalizer(private val pathConverter: FileToPathConverter) : DataExternalizer<File> {
operator fun set(id: Int, file: File) { override fun save(output: DataOutput, file: File) {
storage[id] = pathConverter.toPath(file) EnumeratorStringDescriptor.INSTANCE.save(output, pathConverter.toPath(file))
} }
override fun read(input: DataInput): File {
return pathConverter.toFile(EnumeratorStringDescriptor.INSTANCE.read(input))
}
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt
+23 -18
View File
@@ -59,14 +59,14 @@ open class InMemoryStorage<KEY, VALUE>(
} }
@Synchronized @Synchronized
override fun get(key: KEY): VALUE? = addedEntries[key] ?: modifiedEntries[key] ?: when (key) { override fun get(key: KEY): VALUE? =
in appendedEntries -> { addedEntries[key]
@Suppress("UNCHECKED_CAST") ?: modifiedEntries[key]
((storage[key]!! as Collection<*>) + (appendedEntries[key]!! as Collection<*>)) as VALUE // appendedEntries is handled by AppendableInMemoryStorage#get
} ?: when (key) {
in removedKeys -> null in removedKeys -> null
else -> storage[key] else -> storage[key]
} }
@Synchronized @Synchronized
override fun set(key: KEY, value: VALUE) = when (key) { override fun set(key: KEY, value: VALUE) = when (key) {
@@ -110,7 +110,7 @@ open class InMemoryStorage<KEY, VALUE>(
modifiedEntries.forEach { modifiedEntries.forEach {
storage[it.key] = it.value storage[it.key] = it.value
} }
check(appendedEntries.isEmpty()) { "appendedEntries is not empty" } // appendedEntries is handled by AppendableInMemoryStorage#applyChanges
removedKeys.forEach { removedKeys.forEach {
storage.remove(it) storage.remove(it)
} }
@@ -139,18 +139,23 @@ open class InMemoryStorage<KEY, VALUE>(
} }
/** [InMemoryStorage] where a map entry's value is a [Collection]. */ /** [InMemoryStorage] where a map entry's value is a [Collection] of elements of type [E]. */
@ThreadSafe @ThreadSafe
class AppendableInMemoryStorage<KEY, E, VALUE : Collection<E>>( class AppendableInMemoryStorage<KEY, E>(
private val storage: AppendablePersistentStorage<KEY, E, VALUE>, private val storage: AppendablePersistentStorage<KEY, E>,
) : InMemoryStorage<KEY, VALUE>(storage), AppendablePersistentStorage<KEY, E, VALUE> { ) : InMemoryStorage<KEY, Collection<E>>(storage), AppendablePersistentStorage<KEY, E> {
@Suppress("UNCHECKED_CAST")
@Synchronized @Synchronized
override fun append(key: KEY, elements: VALUE) = when (key) { override fun get(key: KEY): Collection<E>? = when (key) {
in addedEntries -> addedEntries[key] = (addedEntries[key]!! + elements) as VALUE in appendedEntries -> storage[key]!! + appendedEntries[key]!!
in modifiedEntries -> modifiedEntries[key] = (modifiedEntries[key]!! + elements) as VALUE else -> super.get(key)
in appendedEntries -> appendedEntries[key] = (appendedEntries[key]!! + elements) as VALUE }
@Synchronized
override fun append(key: KEY, elements: Collection<E>) = when (key) {
in addedEntries -> addedEntries[key] = addedEntries[key]!! + elements
in modifiedEntries -> modifiedEntries[key] = modifiedEntries[key]!! + elements
in appendedEntries -> appendedEntries[key] = appendedEntries[key]!! + elements
in removedKeys -> { in removedKeys -> {
removedKeys.remove(key) removedKeys.remove(key)
// Note: We update `modifiedEntries` (not `appendedEntries`), because if the entry is removed and then appended, it is // Note: We update `modifiedEntries` (not `appendedEntries`), because if the entry is removed and then appended, it is
build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt
+15 -16
View File
@@ -95,19 +95,19 @@ open class LazyStorage<KEY, VALUE>(
} }
/** [LazyStorage] where a map entry's value is a [Collection]. */ /** [LazyStorage] where a map entry's value is a [Collection] of elements of type [E]. */
@ThreadSafe @ThreadSafe
class AppendableLazyStorage<KEY, E, VALUE : Collection<E>>( class AppendableLazyStorage<KEY, E>(
storageFile: File, storageFile: File,
keyDescriptor: KeyDescriptor<KEY>, keyDescriptor: KeyDescriptor<KEY>,
elementExternalizer: DataExternalizer<E>, elementExternalizer: DataExternalizer<E>,
) : LazyStorage<KEY, VALUE>(storageFile, keyDescriptor, AppendableCollectionExternalizer(elementExternalizer)), ) : LazyStorage<KEY, Collection<E>>(storageFile, keyDescriptor, AppendableCollectionExternalizer(elementExternalizer)),
AppendablePersistentStorage<KEY, E, VALUE> { AppendablePersistentStorage<KEY, E> {
private val appendableCollectionExternalizer = AppendableCollectionExternalizer(elementExternalizer) private val appendableCollectionExternalizer = AppendableCollectionExternalizer(elementExternalizer)
@Synchronized @Synchronized
override fun append(key: KEY, elements: VALUE) { override fun append(key: KEY, elements: Collection<E>) {
getStorageOrCreateNew().appendData( getStorageOrCreateNew().appendData(
key, key,
AppendablePersistentMap.ValueDataAppender { appendableCollectionExternalizer.append(it, elements) } AppendablePersistentMap.ValueDataAppender { appendableCollectionExternalizer.append(it, elements) }
@@ -116,25 +116,25 @@ class AppendableLazyStorage<KEY, E, VALUE : Collection<E>>(
} }
/** /**
* [DataExternalizer] for a [Collection] of type [E]. * [DataExternalizer] for a [Collection] of elements of type [E].
* *
* IMPORTANT: It is a *private* class because it is meant to be used only with a [PersistentHashMap] (e.g., the [read] method reads until * IMPORTANT: It is a *private* class because it is meant to be used only with a [PersistentHashMap] (e.g., the [read] method reads until
* the stream ends, [append] can be called multiple times and its implementation is identical to [save] -- these only work with a * the stream ends, [append] can be called multiple times and its implementation is identical to [save] -- these only work with a
* [PersistentHashMap]). * [PersistentHashMap]).
*/ */
private class AppendableCollectionExternalizer<E, VALUE : Collection<E>>( private class AppendableCollectionExternalizer<E>(
private val elementExternalizer: DataExternalizer<E>, private val elementExternalizer: DataExternalizer<E>,
) : DataExternalizer<VALUE> { ) : DataExternalizer<Collection<E>> {
fun append(output: DataOutput, elements: VALUE) { fun append(output: DataOutput, elements: Collection<E>) {
save(output, elements) save(output, elements)
} }
override fun save(output: DataOutput, value: VALUE) { override fun save(output: DataOutput, value: Collection<E>) {
value.forEach { elementExternalizer.save(output, it) } value.forEach { elementExternalizer.save(output, it) }
} }
override fun read(input: DataInput): VALUE { override fun read(input: DataInput): Collection<E> {
val result = ArrayList<E>() val result = ArrayList<E>()
val stream = input as DataInputStream val stream = input as DataInputStream
@@ -142,8 +142,7 @@ private class AppendableCollectionExternalizer<E, VALUE : Collection<E>>(
result.add(elementExternalizer.read(stream)) result.add(elementExternalizer.read(stream))
} }
@Suppress("UNCHECKED_CAST") return result
return result as VALUE
} }
} }
@@ -164,13 +163,13 @@ fun <KEY, VALUE> createPersistentStorage(
} }
} }
fun <KEY, E, VALUE : Collection<E>> createAppendablePersistentStorage( fun <KEY, E> createAppendablePersistentStorage(
storageFile: File, storageFile: File,
keyDescriptor: KeyDescriptor<KEY>, keyDescriptor: KeyDescriptor<KEY>,
elementExternalizer: DataExternalizer<E>, elementExternalizer: DataExternalizer<E>,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
): AppendablePersistentStorage<KEY, E, VALUE> { ): AppendablePersistentStorage<KEY, E> {
return AppendableLazyStorage<KEY, E, VALUE>(storageFile, keyDescriptor, elementExternalizer).let { storage -> return AppendableLazyStorage(storageFile, keyDescriptor, elementExternalizer).let { storage ->
if (icContext.keepIncrementalCompilationCachesInMemory) { if (icContext.keepIncrementalCompilationCachesInMemory) {
AppendableInMemoryStorage(storage).also { AppendableInMemoryStorage(storage).also {
icContext.transaction.registerInMemoryStorageWrapper(it) icContext.transaction.registerInMemoryStorageWrapper(it)
build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt
+8 -19
View File
@@ -22,26 +22,15 @@ import java.io.File
class LookupMap( class LookupMap(
storage: File, storage: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : ) : AppendableSetBasicMap<LookupSymbolKey, Int>(
AppendableAbstractBasicMap<LookupSymbolKey, Int, Collection<Int>>( storage,
storage, LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache),
LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache), IntExternalizer,
IntExternalizer, icContext,
icContext, ) {
) {
override fun dumpKey(key: LookupSymbolKey): String = key.toString()
override fun dumpValue(value: Collection<Int>): String = value.toString()
@Synchronized
fun add(name: String, scope: String, fileId: Int) { fun add(name: String, scope: String, fileId: Int) {
storage.append(LookupSymbolKey(name, scope), fileId) append(LookupSymbolKey(name, scope), fileId)
} }
override fun get(key: LookupSymbolKey): Collection<Int>? = storage[key]?.toSet()
operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {
storage[key] = fileIds
}
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt
+9 -10
View File
@@ -50,16 +50,15 @@ interface PersistentStorage<KEY, VALUE> : Closeable {
override fun close() override fun close()
} }
/** [PersistentStorage] where a map entry's value is a [Collection]. */ /** [PersistentStorage] where a map entry's value is a [Collection] of elements of type [E]. */
interface AppendablePersistentStorage<KEY, E, VALUE : Collection<E>> : PersistentStorage<KEY, VALUE> { interface AppendablePersistentStorage<KEY, E> : PersistentStorage<KEY, Collection<E>> {
/** Adds the given [elements] to the collection corresponding to the given [key]. */ /** Adds the given [elements] to the collection corresponding to the given [key]. */
fun append(key: KEY, elements: VALUE) fun append(key: KEY, elements: Collection<E>)
/** Adds the given [element] to the collection corresponding to the given [key]. */ /** Adds the given [element] to the collection corresponding to the given [key]. */
fun append(key: KEY, element: E) { fun append(key: KEY, element: E) {
@Suppress("UNCHECKED_CAST") append(key, listOf(element))
append(key, listOf(element) as VALUE)
} }
} }
@@ -111,14 +110,14 @@ abstract class PersistentStorageWrapper<KEY, VALUE>(
} }
} }
/** [PersistentStorageWrapper] where a map entry's value is a [Collection]. */ /** [PersistentStorageWrapper] where a map entry's value is a [Collection] of elements of type [E]. */
@ThreadSafe @ThreadSafe
abstract class AppendablePersistentStorageWrapper<KEY, E, VALUE : Collection<E>>( abstract class AppendablePersistentStorageWrapper<KEY, E>(
private val appendableStorage: AppendablePersistentStorage<KEY, E, VALUE>, private val appendableStorage: AppendablePersistentStorage<KEY, E>,
) : PersistentStorageWrapper<KEY, VALUE>(appendableStorage), AppendablePersistentStorage<KEY, E, VALUE> { ) : PersistentStorageWrapper<KEY, Collection<E>>(appendableStorage), AppendablePersistentStorage<KEY, E> {
@Synchronized @Synchronized
override fun append(key: KEY, elements: VALUE) { override fun append(key: KEY, elements: Collection<E>) {
appendableStorage.append(key, elements) appendableStorage.append(key, elements)
} }
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt
-81
View File
@@ -1,81 +0,0 @@
/*
* Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.utils.ThreadSafe
import java.io.File
/**
* [PersistentStorage] which maps [KEY] to [VALUE] as viewed by the users of this class, but delegates operations to another [storage] which
* maps [INTERNAL_KEY] to [INTERNAL_VALUE].
*
* The users of this class need to provide the transformations from [KEY] to [INTERNAL_KEY], [VALUE] to [INTERNAL_VALUE], and vice versa.
*/
@ThreadSafe
abstract class PersistentStorageAdapter<KEY, VALUE, INTERNAL_KEY, INTERNAL_VALUE>(
private val storage: PersistentStorage<INTERNAL_KEY, INTERNAL_VALUE>,
private val publicToInternalKey: (KEY) -> INTERNAL_KEY,
private val internalToPublicKey: (INTERNAL_KEY) -> KEY,
private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE,
private val internalToPublicValue: (INTERNAL_VALUE) -> VALUE,
) : PersistentStorage<KEY, VALUE>, BasicMap<KEY, VALUE> {
override val storageFile: File = storage.storageFile
@get:Synchronized
override val keys: Set<KEY>
get() = storage.keys.mapTo(LinkedHashSet()) { internalToPublicKey(it) }
@Synchronized
override fun contains(key: KEY): Boolean =
storage.contains(publicToInternalKey(key))
@Synchronized
override fun get(key: KEY): VALUE? =
storage[publicToInternalKey(key)]?.let { internalToPublicValue(it) }
@Synchronized
override fun set(key: KEY, value: VALUE) {
storage[publicToInternalKey(key)] = publicToInternalValue(value)
}
@Synchronized
override fun remove(key: KEY) {
storage.remove(publicToInternalKey(key))
}
@Synchronized
override fun flush() {
storage.close()
}
@Synchronized
override fun close() {
storage.close()
}
}
/** [PersistentStorageAdapter] where a map entry's value is a [Collection]. */
@ThreadSafe
abstract class AppendablePersistentStorageAdapter<KEY, E, VALUE : Collection<E>, INTERNAL_KEY, INTERNAL_E, INTERNAL_VALUE : Collection<INTERNAL_E>>(
private val storage: AppendablePersistentStorage<INTERNAL_KEY, INTERNAL_E, INTERNAL_VALUE>,
private val publicToInternalKey: (KEY) -> INTERNAL_KEY,
internalToPublicKey: (INTERNAL_KEY) -> KEY,
private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE,
internalToPublicValue: (INTERNAL_VALUE) -> VALUE,
) : PersistentStorageAdapter<KEY, VALUE, INTERNAL_KEY, INTERNAL_VALUE>(
storage,
publicToInternalKey,
internalToPublicKey,
publicToInternalValue,
internalToPublicValue
), AppendablePersistentStorage<KEY, E, VALUE> {
@Synchronized
override fun append(key: KEY, elements: VALUE) {
storage.append(publicToInternalKey(key), publicToInternalValue(elements))
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt
+2
View File
@@ -17,6 +17,8 @@ import java.io.File
class RelocatableFileToPathConverter(private val baseDir: File) : FileToPathConverter { class RelocatableFileToPathConverter(private val baseDir: File) : FileToPathConverter {
override fun toPath(file: File): String { override fun toPath(file: File): String {
check(file.isAbsolute) { "Expected absolute path but found relative path: ${file.path}" }
// Note: If the given file is located outside `baseDir`, the relative path will start with "../". It's not "clean", but it can work. // Note: If the given file is located outside `baseDir`, the relative path will start with "../". It's not "clean", but it can work.
// TODO: Re-design the code such that `baseDir` always contains the given file (also add a precondition check here). // TODO: Re-design the code such that `baseDir` always contains the given file (also add a precondition check here).
return file.relativeTo(baseDir).invariantSeparatorsPath return file.relativeTo(baseDir).invariantSeparatorsPath
build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt
+6 -35
View File
@@ -5,44 +5,15 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.EnumeratorStringDescriptor
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File import java.io.File
class SourceToJsOutputMap( class SourceToJsOutputMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AppendableBasicStringMap<String, Collection<String>>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { ) : AppendableSetBasicMap<File, File>(
override fun dumpValue(value: Collection<String>): String = value.dumpCollection() storageFile,
icContext.fileDescriptorForSourceFiles,
@Synchronized icContext.fileDescriptorForOutputFiles,
fun add(key: File, value: File) { icContext
storage.append(pathConverter.toPath(key), pathConverter.toPath(value)) )
}
operator fun get(sourceFile: File): Collection<File> =
storage[pathConverter.toPath(sourceFile)]?.mapTo(mutableSetOf()) { pathConverter.toFile(it) } ?: setOf()
@Synchronized
operator fun set(key: File, values: Collection<File>) {
if (values.isEmpty()) {
remove(key)
return
}
storage[pathConverter.toPath(key)] = values.toSet().map { pathConverter.toPath(it) }
}
@Synchronized
fun remove(key: File) {
storage.remove(pathConverter.toPath(key))
}
@Synchronized
fun removeValues(key: File, removed: Set<File>) {
val notRemoved = this[key].filter { it !in removed }
this[key] = notRemoved
}
}
build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt
+19 -21
View File
@@ -16,11 +16,12 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.DataExternalizer
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.DataInput
import java.io.DataOutput
import java.io.File import java.io.File
internal class SourceToJvmNameMap( internal class SourceToJvmNameMap(
@@ -37,30 +38,27 @@ internal abstract class AbstractSourceToOutputMap<Name>(
private val nameTransformer: NameTransformer<Name>, private val nameTransformer: NameTransformer<Name>,
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : AppendableBasicStringMap<String, Collection<String>>( ) : AppendableSetBasicMap<File, Name>(
storageFile, storageFile,
PathStringDescriptor, icContext.fileDescriptorForSourceFiles,
EnumeratorStringDescriptor.INSTANCE, NameExternalizer(nameTransformer),
icContext icContext
) { ) {
fun clearOutputsForSource(sourceFile: File) {
remove(pathConverter.toPath(sourceFile)) @Synchronized
fun getFqNames(sourceFile: File): Collection<FqName>? =
this[sourceFile]?.map { nameTransformer.asFqName(nameTransformer.asString(it)) }
}
private class NameExternalizer<Name>(private val nameTransformer: NameTransformer<Name>) : DataExternalizer<Name> {
override fun save(output: DataOutput, name: Name) {
StringExternalizer.save(output, nameTransformer.asString(name))
} }
fun add(sourceFile: File, className: Name) { override fun read(input: DataInput): Name {
storage.append(pathConverter.toPath(sourceFile), nameTransformer.asString(className)) return nameTransformer.asName(StringExternalizer.read(input))
} }
fun contains(sourceFile: File): Boolean =
pathConverter.toPath(sourceFile) in storage
operator fun get(sourceFile: File): Collection<Name> =
storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asName)
fun getFqNames(sourceFile: File): Collection<FqName> =
storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asFqName)
override fun dumpValue(value: Collection<String>) =
value.dumpCollection()
} }
build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt
+15 -27
View File
@@ -16,10 +16,8 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import com.intellij.openapi.util.io.FileUtil import com.intellij.util.containers.hash.EqualityPolicy
import com.intellij.openapi.util.text.StringUtil
import com.intellij.util.io.DataExternalizer import com.intellij.util.io.DataExternalizer
import com.intellij.util.io.EnumeratorStringDescriptor
import com.intellij.util.io.IOUtil import com.intellij.util.io.IOUtil
import com.intellij.util.io.KeyDescriptor import com.intellij.util.io.KeyDescriptor
import org.jetbrains.kotlin.inline.InlineFunction import org.jetbrains.kotlin.inline.InlineFunction
@@ -31,10 +29,21 @@ import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.resolve.jvm.JvmClassName
import java.io.* import java.io.*
class DefaultEqualityPolicy<T> : EqualityPolicy<T> {
override fun getHashCode(value: T): Int = value.hashCode()
override fun isEqual(value1: T, value2: T): Boolean = (value1 == value2)
}
fun <T> DataExternalizer<T>.toDescriptor(): KeyDescriptor<T> =
object : KeyDescriptor<T>,
DataExternalizer<T> by this,
EqualityPolicy<T> by DefaultEqualityPolicy<T>() {
}
class LookupSymbolKeyDescriptor( class LookupSymbolKeyDescriptor(
/** If `true`, original values are saved; if `false`, only hashes are saved. */ /** If `true`, original values are saved; if `false`, only hashes are saved. */
private val storeFullFqNames: Boolean = false private val storeFullFqNames: Boolean = false
) : KeyDescriptor<LookupSymbolKey> { ) : KeyDescriptor<LookupSymbolKey>, EqualityPolicy<LookupSymbolKey> by DefaultEqualityPolicy() {
override fun read(input: DataInput): LookupSymbolKey { override fun read(input: DataInput): LookupSymbolKey {
// Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames. // Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames.
@@ -67,10 +76,6 @@ class LookupSymbolKeyDescriptor(
output.writeInt(value.scopeHash) output.writeInt(value.scopeHash)
} }
} }
override fun getHashCode(value: LookupSymbolKey): Int = value.hashCode()
override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2
} }
object FqNameExternalizer : DataExternalizer<FqName> { object FqNameExternalizer : DataExternalizer<FqName> {
@@ -220,21 +225,6 @@ object StringExternalizer : DataExternalizer<String> {
override fun read(input: DataInput): String = IOUtil.readString(input) override fun read(input: DataInput): String = IOUtil.readString(input)
} }
object PathStringDescriptor : EnumeratorStringDescriptor() {
override fun getHashCode(path: String): Int {
return if (StringUtil.isEmpty(path)) 0 else FileUtil.toCanonicalPath(path).hashCode()
}
override fun isEqual(val1: String, val2: String?): Boolean {
if (val1 == val2) return true
if (val2 == null) return false
val path1 = FileUtil.toCanonicalPath(val1)
val path2 = FileUtil.toCanonicalPath(val2)
return path1 == path2
}
}
/** [DataExternalizer] that delegates to another [DataExternalizer] depending on the type of the object to externalize. */ /** [DataExternalizer] that delegates to another [DataExternalizer] depending on the type of the object to externalize. */
class DelegateDataExternalizer<T>( class DelegateDataExternalizer<T>(
val types: List<Class<out T>>, val types: List<Class<out T>>,
@@ -328,7 +318,7 @@ private class CollectionExternalizerForPersistentHashMap<T>(
* (see [CollectionExternalizerForPersistentHashMap]). * (see [CollectionExternalizerForPersistentHashMap]).
* *
* Currently, we can't change the name or implementation of this class because it is still used by the `compiler-reference-index` module in * Currently, we can't change the name or implementation of this class because it is still used by the `compiler-reference-index` module in
* the Kotlin IDEA plugin and that code relies on this name and implementation being unchanged (see KT-62288). * the Kotlin IDEA plugin and that code relies on this name and implementation being unchanged (see KTIJ-27258).
* *
* Once we remove that dependency, we can remove this class. * Once we remove that dependency, we can remove this class.
*/ */
@@ -345,7 +335,7 @@ object IntCollectionExternalizer :
/** [DataExternalizer] for a [Collection]. */ /** [DataExternalizer] for a [Collection]. */
open class CollectionExternalizerV2<T, C : Collection<T>>( open class CollectionExternalizerV2<T, C : Collection<T>>(
private val elementExternalizer: DataExternalizer<T>, private val elementExternalizer: DataExternalizer<T>,
private val newCollection: (size: Int) -> MutableCollection<T>, private val newCollection: (size: Int) -> MutableCollection<T> = { size -> ArrayList(size) },
) : DataExternalizer<C> { ) : DataExternalizer<C> {
override fun save(output: DataOutput, collection: C) { override fun save(output: DataOutput, collection: C) {
@@ -369,8 +359,6 @@ open class CollectionExternalizerV2<T, C : Collection<T>>(
} }
} }
object StringCollectionExternalizer : CollectionExternalizerV2<String, Collection<String>>(StringExternalizer, { size -> ArrayList(size) })
class ListExternalizer<T>(elementExternalizer: DataExternalizer<T>) : class ListExternalizer<T>(elementExternalizer: DataExternalizer<T>) :
CollectionExternalizerV2<T, List<T>>(elementExternalizer, { size -> ArrayList(size) }) CollectionExternalizerV2<T, List<T>>(elementExternalizer, { size -> ArrayList(size) })
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt
+1 -8
View File
@@ -50,20 +50,13 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
val closer = Closer.create() val closer = Closer.create()
caches.forEach { caches.forEach {
closer.register(CacheCloser(it)) closer.register(it)
} }
closer.close() closer.close()
isClosed = true isClosed = true
} }
private class CacheCloser(private val cache: BasicMapsOwner) : Closeable {
override fun close() {
cache.close()
}
}
} }
open class IncrementalJvmCachesManager( open class IncrementalJvmCachesManager(
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCompilerRunner.kt
+3 -3
View File
@@ -38,8 +38,8 @@ import org.jetbrains.kotlin.config.Services
import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.ExpectActualTracker
import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.incremental.parsing.classesFqNames import org.jetbrains.kotlin.incremental.parsing.classesFqNames
import org.jetbrains.kotlin.incremental.storage.BasicFileToPathConverter
import org.jetbrains.kotlin.incremental.storage.FileLocations import org.jetbrains.kotlin.incremental.storage.FileLocations
import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter
import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector import org.jetbrains.kotlin.incremental.util.BufferingMessageCollector
import org.jetbrains.kotlin.incremental.util.ExceptionLocation import org.jetbrains.kotlin.incremental.util.ExceptionLocation
import org.jetbrains.kotlin.incremental.util.reportException import org.jetbrains.kotlin.incremental.util.reportException
@@ -90,8 +90,8 @@ abstract class IncrementalCompilerRunner<
fileLocations: FileLocations?, fileLocations: FileLocations?,
transaction: CompilationTransaction, transaction: CompilationTransaction,
) = IncrementalCompilationContext( ) = IncrementalCompilationContext(
pathConverterForSourceFiles = fileLocations?.let { it.getRelocatablePathConverterForSourceFiles() } ?: FileToAbsolutePathConverter, pathConverterForSourceFiles = fileLocations?.let { it.getRelocatablePathConverterForSourceFiles() } ?: BasicFileToPathConverter,
pathConverterForOutputFiles = fileLocations?.let { it.getRelocatablePathConverterForOutputFiles() } ?: FileToAbsolutePathConverter, pathConverterForOutputFiles = fileLocations?.let { it.getRelocatablePathConverterForOutputFiles() } ?: BasicFileToPathConverter,
transaction = transaction, transaction = transaction,
reporter = reporter, reporter = reporter,
trackChangesInLookupCache = shouldTrackChangesInLookupCache, trackChangesInLookupCache = shouldTrackChangesInLookupCache,
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/InputsCache.kt
+2 -2
View File
@@ -52,7 +52,7 @@ class InputsCache(
// generatedFiles can contain multiple entries with the same source file // generatedFiles can contain multiple entries with the same source file
// for example Kapt3 IC will generate a .java stub and .class stub for each source file // for example Kapt3 IC will generate a .java stub and .class stub for each source file
fun registerOutputForSourceFiles(generatedFiles: List<GeneratedFile>) { fun registerOutputForSourceFiles(generatedFiles: List<GeneratedFile>) {
val sourceToOutput = MultiMap<File, File>() val sourceToOutput = MultiMap.createLinked<File, File>()
for (generatedFile in generatedFiles) { for (generatedFile in generatedFiles) {
for (source in generatedFile.sourceFiles) { for (source in generatedFile.sourceFiles) {
@@ -61,7 +61,7 @@ class InputsCache(
} }
for ((source, outputs) in sourceToOutput.entrySet()) { for ((source, outputs) in sourceToOutput.entrySet()) {
sourceToOutputMap[source] = outputs sourceToOutputMap[source] = outputs.toSet()
} }
} }
} }
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt
+15 -17
View File
@@ -18,41 +18,39 @@ package org.jetbrains.kotlin.incremental.snapshots
import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.ChangedFiles
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.storage.BasicStringMap import org.jetbrains.kotlin.incremental.storage.AbstractBasicMap
import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor
import java.io.File import java.io.File
import java.util.*
class FileSnapshotMap( class FileSnapshotMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<FileSnapshot>(storageFile, PathStringDescriptor, FileSnapshotExternalizer, icContext) { ) : AbstractBasicMap<File, FileSnapshot>(
storageFile,
override fun dumpValue(value: FileSnapshot): String = icContext.fileDescriptorForSourceFiles,
value.toString() FileSnapshotExternalizer,
icContext
) {
@Synchronized @Synchronized
fun compareAndUpdate(newFiles: Iterable<File>): ChangedFiles.Known { fun compareAndUpdate(newFiles: Iterable<File>): ChangedFiles.Known {
val snapshotProvider = SimpleFileSnapshotProviderImpl() val snapshotProvider = SimpleFileSnapshotProviderImpl()
val newOrModified = ArrayList<File>() val newOrModified = ArrayList<File>()
val removed = ArrayList<File>() val removed = ArrayList<File>()
val newPaths = newFiles.mapTo(HashSet(), transform = pathConverter::toPath) val newFilesSet = newFiles.toSet()
for (oldPath in storage.keys) { for (oldFile in keys) {
if (oldPath !in newPaths) { if (oldFile !in newFilesSet) {
storage.remove(oldPath) remove(oldFile)
removed.add(pathConverter.toFile(oldPath)) removed.add(oldFile)
} }
} }
for (path in newPaths) { for (file in newFilesSet) {
val file = pathConverter.toFile(path) val oldSnapshot = this[file]
val oldSnapshot = storage[path]
val newSnapshot = snapshotProvider[file] val newSnapshot = snapshotProvider[file]
if (oldSnapshot == null || oldSnapshot != newSnapshot) { if (oldSnapshot == null || oldSnapshot != newSnapshot) {
newOrModified.add(file) newOrModified.add(file)
storage[path] = newSnapshot this[file] = newSnapshot
} }
} }
compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt
+12 -15
View File
@@ -6,24 +6,21 @@
package org.jetbrains.kotlin.incremental.storage package org.jetbrains.kotlin.incremental.storage
import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
import org.jetbrains.kotlin.incremental.dumpCollection
import java.io.File import java.io.File
class SourceToOutputFilesMap( class SourceToOutputFilesMap(
storageFile: File, storageFile: File,
icContext: IncrementalCompilationContext, icContext: IncrementalCompilationContext,
) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { ) : AppendableSetBasicMap<File, File>(
operator fun set(sourceFile: File, outputFiles: Collection<File>) { storageFile,
storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] = icContext.fileDescriptorForSourceFiles,
outputFiles.toSet().map(icContext.pathConverterForOutputFiles::toPath) icContext.fileDescriptorForOutputFiles,
icContext
) {
@Synchronized
fun getAndRemove(sourceFile: File): Set<File>? {
return get(sourceFile).also {
remove(sourceFile)
}
} }
}
operator fun get(sourceFile: File): Collection<File>? =
storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)]?.map(icContext.pathConverterForOutputFiles::toFile)
override fun dumpValue(value: Collection<String>) =
value.dumpCollection()
fun getAndRemove(file: File): Collection<File>? =
get(file).also { storage.remove(icContext.pathConverterForSourceFiles.toPath(file)) }
}
compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt
+21 -17
View File
@@ -60,49 +60,53 @@ class SourceToOutputFilesMapTest {
@Test @Test
fun testSetOneGetReturnsOne() { fun testSetOneGetReturnsOne() {
stofMap[fooDotKt] = listOf(fooDotClass) stofMap[fooDotKt] = setOf(fooDotClass)
assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) assertEquals(setOf(fooDotClass), stofMap[fooDotKt])
} }
@Test @Test
fun testSetDupeGetReturnsUnique() { fun testSetDupeGetReturnsUnique() {
stofMap[fooDotKt] = listOf(fooDotClass, fooDotClass) stofMap.append(fooDotKt, fooDotClass)
stofMap.append(fooDotKt, fooDotClass)
assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) assertEquals(setOf(fooDotClass), stofMap[fooDotKt])
} }
@Test @Test
fun testSetOverwriteGetReturnsNew() { fun testSetOverwriteGetReturnsNew() {
val fooKtDotClass = classesDir.resolve("FooKt.class") val fooKtDotClass = classesDir.resolve("FooKt.class")
stofMap[fooDotKt] = listOf(fooDotClass) stofMap[fooDotKt] = setOf(fooDotClass)
stofMap[fooDotKt] = listOf(fooKtDotClass) stofMap[fooDotKt] = setOf(fooKtDotClass)
assertEquals(listOf(fooKtDotClass), stofMap[fooDotKt]) assertEquals(setOf(fooKtDotClass), stofMap[fooDotKt])
} }
@Test @Test
fun testSetRelativeFails() { fun testSetRelativePathFails() {
assertFailsWith<IllegalArgumentException> { assertFailsWith<IllegalStateException> {
stofMap[fooDotKt] = listOf(File("relativePath")) stofMap[fooDotKt] = setOf(File("relativePath"))
}
assertFailsWith<IllegalStateException> {
stofMap[File("relativePath")] = setOf(fooDotClass)
} }
} }
@Test @Test
fun testGetRelativeFails() { fun testGetRelativePathFails() {
stofMap[fooDotKt] = listOf(fooDotClass) stofMap[fooDotKt] = setOf(fooDotClass)
assertFailsWith<IllegalArgumentException> { assertFailsWith<IllegalStateException> {
stofMap[fooDotKt.relativeTo(srcDir)] stofMap[File("relativePath")]
} }
} }
@Test @Test
fun testGetAndRemove() { fun testGetAndRemove() {
stofMap[fooDotKt] = listOf(fooDotClass) stofMap[fooDotKt] = setOf(fooDotClass)
assertEquals(listOf(fooDotClass), stofMap.getAndRemove(fooDotKt)) assertEquals(setOf(fooDotClass), stofMap.getAndRemove(fooDotKt))
assertNull(stofMap[fooDotKt]) assertNull(stofMap[fooDotKt])
} }
} }
repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt
-1
View File
@@ -218,7 +218,6 @@ class CodeConformanceTest : TestCase() {
"These files are affected:\n%s", "These files are affected:\n%s",
allowedFiles = listOf( allowedFiles = listOf(
"analysis/light-classes-base/src/org/jetbrains/kotlin/asJava/classes/KotlinClassInnerStuffCache.kt", "analysis/light-classes-base/src/org/jetbrains/kotlin/asJava/classes/KotlinClassInnerStuffCache.kt",
"build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt",
"compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/CliVirtualFileFinder.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/CliVirtualFileFinder.kt",
"compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCliJavaFileManagerImpl.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCliJavaFileManagerImpl.kt",
"compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/index/JvmDependenciesIndexImpl.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/index/JvmDependenciesIndexImpl.kt",