From 5562c95155f30f7ff3e7bd33cd17a49bcc778890 Mon Sep 17 00:00:00 2001 From: Hung Nguyen Date: Wed, 15 Nov 2023 12:22:40 +0000 Subject: [PATCH] [IC] Refactor IC maps to reuse code and ensure consistency - Part 2 In commit 4e89dcf, we have prepared the API for IC maps in top interfaces and provide the implementation in abstract classes. In this commit, we refactor IC maps so that they directly inherit/reuse the implementation from the superclasses without having to reimplement the APIs for a map. Test: Existing tests (refactoring change) ^KT-63456: In progress Authored-by: Hung Nguyen Merge-request: KOTLIN-MR-801 Merged-by: Evgenii Mazhukin --- .../incremental/AbstractIncrementalCache.kt | 48 +++--- .../IncrementalCompilationContext.kt | 15 +- .../kotlin/incremental/IncrementalJsCache.kt | 84 +++++----- .../kotlin/incremental/IncrementalJvmCache.kt | 88 ++++------- .../kotlin/incremental/LookupStorage.kt | 30 ++-- .../kotlin/incremental/storage/BasicMap.kt | 144 +++++++++++++----- .../incremental/storage/ClassOneToManyMap.kt | 60 ++++---- .../storage/ComplementarySourceFilesMap.kt | 26 +--- .../kotlin/incremental/storage/FileToIdMap.kt | 30 +--- .../storage/FileToPathConverter.kt | 52 +++++-- .../kotlin/incremental/storage/IdToFileMap.kt | 33 +++- .../incremental/storage/InMemoryStorage.kt | 41 ++--- .../kotlin/incremental/storage/LazyStorage.kt | 31 ++-- .../kotlin/incremental/storage/LookupMap.kt | 27 +--- .../incremental/storage/PersistentStorage.kt | 19 ++- .../storage/PersistentStorageAdapter.kt | 81 ---------- .../storage/RelocatableFileToPathConverter.kt | 2 + .../storage/SourceToJsOutputMap.kt | 41 +---- .../incremental/storage/SourceToOutputMaps.kt | 40 +++-- .../incremental/storage/externalizers.kt | 42 ++--- .../incremental/IncrementalCachesManager.kt | 9 +- .../incremental/IncrementalCompilerRunner.kt | 6 +- .../kotlin/incremental/InputsCache.kt | 4 +- .../incremental/snapshots/FileSnapshotMap.kt | 32 ++-- .../storage/SourceToOutputFilesMap.kt | 27 ++-- .../storage/SourceToOutputFilesMapTest.kt | 38 ++--- .../kotlin/code/CodeConformanceTest.kt | 1 - 27 files changed, 473 insertions(+), 578 deletions(-) delete mode 100644 build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt diff --git a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt index 90a8cf32cbd..f67df61b82a 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/AbstractIncrementalCache.kt @@ -16,7 +16,6 @@ package org.jetbrains.kotlin.incremental -import com.intellij.util.io.EnumeratorStringDescriptor import org.jetbrains.kotlin.incremental.storage.* import org.jetbrains.kotlin.metadata.ProtoBuf import org.jetbrains.kotlin.metadata.deserialization.Flags @@ -26,7 +25,6 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.serialization.deserialization.getClassId import java.io.File import java.util.* -import kotlin.collections.HashSet /** * Incremental cache common for JVM and JS, ClassName type aware @@ -94,13 +92,13 @@ abstract class AbstractIncrementalCache( private val complementaryFilesMap = registerMap(ComplementarySourceFilesMap(COMPLEMENTARY_FILES.storageFile, icContext)) override fun classesFqNamesBySources(files: Iterable): Collection = - files.flatMapTo(HashSet()) { sourceToClassesMap.getFqNames(it) } + files.flatMapTo(mutableSetOf()) { sourceToClassesMap.getFqNames(it).orEmpty() } override fun getSubtypesOf(className: FqName): Sequence = - subtypesMap[className].asSequence() + subtypesMap[className].orEmpty().asSequence() override fun getSupertypesOf(className: FqName): Sequence { - return supertypesMap[className].asSequence() + return supertypesMap[className].orEmpty().asSequence() } override fun isSealed(className: FqName): Boolean? { @@ -112,10 +110,10 @@ abstract class AbstractIncrementalCache( override fun markDirty(removedAndCompiledSources: Collection) { for (sourceFile in removedAndCompiledSources) { - sourceToClassesMap[sourceFile].forEach { className -> + sourceToClassesMap[sourceFile]?.forEach { className -> markDirty(className) } - sourceToClassesMap.clearOutputsForSource(sourceFile) + sourceToClassesMap.remove(sourceFile) } } @@ -137,9 +135,9 @@ abstract class AbstractIncrementalCache( .toSet() val child = nameResolver.getClassId(proto.fqName).asSingleFqName() - parents.forEach { subtypesMap.add(it, child) } + parents.forEach { subtypesMap.append(it, child) } - val removedSupertypes = supertypesMap[child].filter { it !in parents } + val removedSupertypes = supertypesMap[child].orEmpty().filter { it !in parents } removedSupertypes.forEach { subtypesMap.removeValues(it, setOf(child)) } supertypesMap[child] = parents @@ -163,8 +161,8 @@ abstract class AbstractIncrementalCache( val childrenFqNames = hashSetOf() for (removedFqName in removedFqNames) { - parentsFqNames.addAll(cache.supertypesMap[removedFqName]) - childrenFqNames.addAll(cache.subtypesMap[removedFqName]) + parentsFqNames.addAll(cache.supertypesMap[removedFqName].orEmpty()) + childrenFqNames.addAll(cache.subtypesMap[removedFqName].orEmpty()) cache.supertypesMap.remove(removedFqName) cache.subtypesMap.remove(removedFqName) @@ -188,20 +186,12 @@ abstract class AbstractIncrementalCache( protected class ClassFqNameToSourceMap( storageFile: File, icContext: IncrementalCompilationContext, - ) : BasicStringMap(storageFile, EnumeratorStringDescriptor(), PathStringDescriptor, icContext) { - operator fun set(fqName: FqName, sourceFile: File) { - storage[fqName.asString()] = pathConverter.toPath(sourceFile) - } - - operator fun get(fqName: FqName): File? = - storage[fqName.asString()]?.let(pathConverter::toFile) - - fun remove(fqName: FqName) { - storage.remove(fqName.asString()) - } - - override fun dumpValue(value: String) = value - } + ) : AbstractBasicMap( + storageFile, + FqNameExternalizer.toDescriptor(), + icContext.fileDescriptorForSourceFiles, + icContext + ) override fun getComplementaryFilesRecursive(dirtyFiles: Collection): Collection { val complementaryFiles = HashSet() @@ -216,10 +206,10 @@ abstract class AbstractIncrementalCache( continue } processedFiles.add(file) - complementaryFilesMap[file].forEach { + complementaryFilesMap[file]?.forEach { if (complementaryFiles.add(it) && !processedFiles.contains(it)) filesQueue.add(it) } - val classes2recompile = sourceToClassesMap.getFqNames(file) + val classes2recompile = sourceToClassesMap.getFqNames(file).orEmpty() classes2recompile.filter { !processedClasses.contains(it) }.forEach { class2recompile -> processedClasses.add(class2recompile) val sealedClasses = findSealedSupertypes(class2recompile, listOf(this)) @@ -248,11 +238,11 @@ abstract class AbstractIncrementalCache( for (actual in actuals) { actualToExpect.getOrPut(actual) { hashSetOf() }.add(expect) } - complementaryFilesMap[expect] = actuals.union(complementaryFilesMap[expect]) + complementaryFilesMap[expect] = actuals.union(complementaryFilesMap[expect].orEmpty()) } for ((actual, expects) in actualToExpect) { - complementaryFilesMap[actual] = expects.union(complementaryFilesMap[actual]) + complementaryFilesMap[actual] = expects.union(complementaryFilesMap[actual].orEmpty()) } } } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt index 65491d901b4..f967402017a 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt @@ -5,15 +5,17 @@ package org.jetbrains.kotlin.incremental +import com.intellij.util.io.KeyDescriptor import org.jetbrains.kotlin.build.report.DoNothingICReporter import org.jetbrains.kotlin.build.report.ICReporter -import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter +import org.jetbrains.kotlin.incremental.storage.BasicFileToPathConverter import org.jetbrains.kotlin.incremental.storage.FileToPathConverter +import java.io.File class IncrementalCompilationContext( - // The root directories of source files and output files are different, so we may need different `FileToPathConverter`s - val pathConverterForSourceFiles: FileToPathConverter = FileToAbsolutePathConverter, - val pathConverterForOutputFiles: FileToPathConverter = FileToAbsolutePathConverter, + // The root directories of source files and output files are different, so we need different `FileToPathConverter`s + val pathConverterForSourceFiles: FileToPathConverter = BasicFileToPathConverter, + val pathConverterForOutputFiles: FileToPathConverter = BasicFileToPathConverter, val storeFullFqNamesInLookupCache: Boolean = false, val transaction: CompilationTransaction = NonRecoverableCompilationTransaction(), val reporter: ICReporter = DoNothingICReporter, @@ -46,7 +48,6 @@ class IncrementalCompilationContext( keepIncrementalCompilationCachesInMemory ) - // FIXME: Remove `pathConverter` and require its users to decide whether to use `pathConverterForSourceFiles` or - // `pathConverterForOutputFiles` - val pathConverter = pathConverterForSourceFiles + val fileDescriptorForSourceFiles: KeyDescriptor = pathConverterForSourceFiles.getFileDescriptor() + val fileDescriptorForOutputFiles: KeyDescriptor = pathConverterForOutputFiles.getFileDescriptor() } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt index 89df06d1af8..f2ee6fdfaee 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt @@ -17,6 +17,7 @@ package org.jetbrains.kotlin.incremental import com.intellij.util.io.DataExternalizer +import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.build.GeneratedFile import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumerImpl import org.jetbrains.kotlin.incremental.js.IrTranslationResultValue @@ -79,7 +80,7 @@ open class IncrementalJsCache( removedAndCompiledSources.forEach { sourceFile -> sourceToJsOutputsMap.remove(sourceFile) // The common prefix of all FQN parents has to be the file package - sourceToClassesMap[sourceFile].map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let { + sourceToClassesMap[sourceFile].orEmpty().map { it.parentOrNull()?.asString() ?: "" }.minByOrNull { it.length }?.let { packageMetadata.remove(it) } } @@ -99,7 +100,7 @@ open class IncrementalJsCache( } fun getOutputsBySource(sourceFile: File): Collection { - return sourceToJsOutputsMap[sourceFile] + return sourceToJsOutputsMap[sourceFile].orEmpty() } fun compareAndUpdate(incrementalResults: IncrementalResultsConsumerImpl, changesCollector: ChangesCollector) { @@ -132,7 +133,7 @@ open class IncrementalJsCache( } for ((packageName, metadata) in incrementalResults.packageMetadata) { - packageMetadata.put(packageName, metadata) + packageMetadata[packageName] = metadata } for ((srcFile, irData) in incrementalResults.irFileData) { @@ -142,7 +143,7 @@ open class IncrementalJsCache( } private fun registerOutputForFile(srcFile: File, name: FqName) { - sourceToClassesMap.add(srcFile, name) + sourceToClassesMap.append(srcFile, name) dirtyOutputClassesMap.notDirty(name) } @@ -159,7 +160,7 @@ open class IncrementalJsCache( fun nonDirtyPackageParts(): Map = hashMapOf().apply { - for (file in translationResults.keys()) { + for (file in translationResults.keys) { if (file !in dirtySources) { put(file, translationResults[file]!!) @@ -175,7 +176,7 @@ open class IncrementalJsCache( fun nonDirtyIrParts(): Map = hashMapOf().apply { - for (file in irTranslationResults.keys()) { + for (file in irTranslationResults.keys) { if (file !in dirtySources) { put(file, irTranslationResults[file]!!) @@ -189,7 +190,7 @@ open class IncrementalJsCache( for (generatedFile in generatedFiles) { for (source in generatedFile.sourceFiles) { if (dirtySources.contains(source)) - sourceToJsOutputsMap.add(source, generatedFile.outputFile) + sourceToJsOutputsMap.append(source, generatedFile.outputFile) } } } @@ -228,34 +229,32 @@ private class TranslationResultMap( storageFile: File, private val protoData: ProtoDataProvider, icContext: IncrementalCompilationContext, -) : - BasicStringMap(storageFile, TranslationResultValueExternalizer, icContext) { +) : AbstractBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + TranslationResultValueExternalizer, + icContext +) { + + @TestOnly override fun dumpValue(value: TranslationResultValue): String = "Metadata: ${value.metadata.md5()}, Binary AST: ${value.binaryAst.md5()}, InlineData: ${value.inlineData.md5()}" @Synchronized fun put(sourceFile: File, newMetadata: ByteArray, newBinaryAst: ByteArray, newInlineData: ByteArray) { - storage[pathConverter.toPath(sourceFile)] = + this[sourceFile] = TranslationResultValue(metadata = newMetadata, binaryAst = newBinaryAst, inlineData = newInlineData) } - @Synchronized - operator fun get(sourceFile: File): TranslationResultValue? = - storage[pathConverter.toPath(sourceFile)] - - fun keys(): Collection = - storage.keys.map { pathConverter.toFile(it) } - @Synchronized fun remove(sourceFile: File, changesCollector: ChangesCollector) { - val path = pathConverter.toPath(sourceFile) - val protoBytes = storage[path]!!.metadata + val protoBytes = this[sourceFile]!!.metadata val protoMap = protoData(sourceFile, protoBytes) for ((_, protoData) in protoMap) { changesCollector.collectProtoChanges(oldData = protoData, newData = null) } - storage.remove(path) + remove(sourceFile) } } @@ -313,8 +312,14 @@ private object IrTranslationResultValueExternalizer : DataExternalizer(storageFile, IrTranslationResultValueExternalizer, icContext) { +) : AbstractBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + IrTranslationResultValueExternalizer, + icContext +) { + + @TestOnly override fun dumpValue(value: IrTranslationResultValue): String = "Filedata: ${value.fileData.md5()}, " + "Types: ${value.types.md5()}, " + @@ -323,6 +328,7 @@ private class IrTranslationResultMap( "Declarations: ${value.declarations.md5()}, " + "Bodies: ${value.bodies.md5()}" + @Synchronized fun put( sourceFile: File, newFiledata: ByteArray, @@ -335,20 +341,9 @@ private class IrTranslationResultMap( newFileMetadata: ByteArray, debugInfos: ByteArray?, ) { - storage[pathConverter.toPath(sourceFile)] = + this[sourceFile] = IrTranslationResultValue(newFiledata, newTypes, newSignatures, newStrings, newDeclarations, newBodies, fqn, newFileMetadata, debugInfos) } - - operator fun get(sourceFile: File): IrTranslationResultValue? = - storage[pathConverter.toPath(sourceFile)] - - fun keys(): Collection = - storage.keys.map { pathConverter.toFile(it) } - - fun remove(sourceFile: File) { - val path = pathConverter.toPath(sourceFile) - storage.remove(path) - } } private class ProtoDataProvider(private val serializerProtocol: SerializerExtensionProtocol) { @@ -396,16 +391,21 @@ fun getProtoData(sourceFile: File, metadata: ByteArray): Map private class InlineFunctionsMap( storageFile: File, icContext: IncrementalCompilationContext, -) : BasicStringMap>(storageFile, StringToLongMapExternalizer, icContext) { +) : AbstractBasicMap>( + storageFile, + icContext.fileDescriptorForSourceFiles, + StringToLongMapExternalizer, + icContext +) { + @Synchronized fun process(srcFile: File, newMap: Map, changesCollector: ChangesCollector) { - val key = pathConverter.toPath(srcFile) - val oldMap = storage[key] ?: emptyMap() + val oldMap = this[srcFile] ?: emptyMap() if (newMap.isNotEmpty()) { - storage[key] = newMap + this[srcFile] = newMap } else { - storage.remove(key) + remove(srcFile) } for (fn in oldMap.keys + newMap.keys) { @@ -415,11 +415,7 @@ private class InlineFunctionsMap( } } - @Synchronized - fun remove(sourceFile: File) { - storage.remove(pathConverter.toPath(sourceFile)) - } - + @TestOnly override fun dumpValue(value: Map): String = value.dumpMap { java.lang.Long.toHexString(it) } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt index ada7dd9a3e8..ecc35c44de5 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt @@ -17,10 +17,7 @@ package org.jetbrains.kotlin.incremental import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName -import com.intellij.util.containers.CollectionFactory import com.intellij.util.io.BooleanDataDescriptor -import com.intellij.util.io.EnumeratorStringDescriptor -import gnu.trove.THashSet import org.jetbrains.annotations.TestOnly import org.jetbrains.kotlin.build.GeneratedJvmClass import org.jetbrains.kotlin.incremental.storage.* @@ -88,13 +85,10 @@ open class IncrementalJvmCache( // used in gradle @Suppress("unused") fun classesBySources(sources: Iterable): Iterable = - sources.flatMap { sourceToClassesMap[it] } - - fun sourceInCache(file: File): Boolean = - sourceToClassesMap.contains(file) + sources.flatMap { sourceToClassesMap[it].orEmpty() } fun sourcesByInternalName(internalName: String): Collection = - internalNameToSource.getFiles(internalName) + internalNameToSource[internalName].orEmpty() fun getAllPartsOfMultifileFacade(facade: JvmClassName): Collection? { return multifileFacadeToParts[facade] @@ -111,7 +105,7 @@ open class IncrementalJvmCache( val jvmClassName = JvmClassName.byInternalName(MODULE_MAPPING_FILE_NAME) protoMap.storeModuleMapping(jvmClassName, file.readBytes()) dirtyOutputClassesMap.notDirty(jvmClassName) - sourceFiles.forEach { sourceToClassesMap.add(it, jvmClassName) } + sourceFiles.forEach { sourceToClassesMap.append(it, jvmClassName) } } open fun saveFileToCache(generatedClass: GeneratedJvmClass, changesCollector: ChangesCollector) { @@ -133,7 +127,7 @@ open class IncrementalJvmCache( if (sourceFiles != null) { sourceFiles.forEach { - sourceToClassesMap.add(it, className) + sourceToClassesMap.append(it, className) } internalNameToSource[className.internalName] = sourceFiles } @@ -176,7 +170,7 @@ open class IncrementalJvmCache( assert(sourceFiles.size == 1) { "Multifile class part from several source files: $sourceFiles" } } packagePartMap.addPackagePart(className) - partToMultifileFacade.set(className.internalName, kotlinClassInfo.multifileClassName!!) + partToMultifileFacade[className] = kotlinClassInfo.multifileClassName!! protoMap.process(kotlinClassInfo, changesCollector) constantsMap.process(kotlinClassInfo, changesCollector) @@ -235,7 +229,7 @@ open class IncrementalJvmCache( fun saveJavaClassProto(source: File?, serializedJavaClass: SerializedJavaClass, collector: ChangesCollector) { val jvmClassName = JvmClassName.byClassId(serializedJavaClass.classId) javaSourcesProtoMap.process(jvmClassName, serializedJavaClass, collector) - source?.let { sourceToClassesMap.add(source, jvmClassName) } + source?.let { sourceToClassesMap.append(source, jvmClassName) } addToClassStorage(serializedJavaClass.toProtoData(), source) // collector.addJavaProto(ClassProtoData(proto, nameResolver)) dirtyOutputClassesMap.notDirty(jvmClassName) @@ -263,7 +257,7 @@ open class IncrementalJvmCache( val facadesWithRemovedParts = hashMapOf>() for (dirtyClass in dirtyClasses) { - val facade = partToMultifileFacade.get(dirtyClass) ?: continue + val facade = partToMultifileFacade[dirtyClass] ?: continue val facadeClassName = JvmClassName.byInternalName(facade) val removedParts = facadesWithRemovedParts.getOrPut(facadeClassName) { hashSetOf() } removedParts.add(dirtyClass.internalName) @@ -311,7 +305,7 @@ open class IncrementalJvmCache( override fun getObsoleteMultifileClasses(): Collection { val obsoleteMultifileClasses = linkedSetOf() for (dirtyClass in dirtyOutputClassesMap.getDirtyOutputClasses()) { - val dirtyFacade = partToMultifileFacade.get(dirtyClass) ?: continue + val dirtyFacade = partToMultifileFacade[dirtyClass] ?: continue obsoleteMultifileClasses.add(dirtyFacade) } debugLog("Obsolete multifile class facades: $obsoleteMultifileClasses") @@ -514,59 +508,32 @@ open class IncrementalJvmCache( private inner class MultifileClassFacadeMap( storageFile: File, icContext: IncrementalCompilationContext, - ) : - BasicStringMap>(storageFile, StringCollectionExternalizer, icContext) { - - @Synchronized - operator fun set(className: JvmClassName, partNames: Collection) { - storage[className.internalName] = partNames - } - - operator fun get(className: JvmClassName): Collection? = - storage[className.internalName] - - operator fun contains(className: JvmClassName): Boolean = - className.internalName in storage - - @Synchronized - fun remove(className: JvmClassName) { - storage.remove(className.internalName) - } - - override fun dumpValue(value: Collection): String = value.dumpCollection() - } + ) : AppendableBasicMap( + storageFile, + JvmClassNameExternalizer.toDescriptor(), + StringExternalizer, + icContext + ) private inner class MultifileClassPartMap( storageFile: File, icContext: IncrementalCompilationContext, - ) : - BasicStringMap(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { - - fun get(partName: JvmClassName): String? = - storage[partName.internalName] - - @Synchronized - fun remove(className: JvmClassName) { - storage.remove(className.internalName) - } - - override fun dumpValue(value: String): String = value - } + ) : AbstractBasicMap( + storageFile, + JvmClassNameExternalizer.toDescriptor(), + StringExternalizer, + icContext + ) inner class InternalNameToSourcesMap( storageFile: File, icContext: IncrementalCompilationContext, - ) : BasicStringMap>(storageFile, EnumeratorStringDescriptor(), PathCollectionExternalizer, icContext) { - operator fun set(internalName: String, sourceFiles: Collection) { - storage[internalName] = pathConverter.toPaths(sourceFiles) - } - - fun getFiles(internalName: String): Collection = - pathConverter.toFiles(storage[internalName] ?: emptyList()) - - override fun dumpValue(value: Collection): String = - value.dumpCollection() - } + ) : AppendableBasicMap( + storageFile, + StringExternalizer.toDescriptor(), + icContext.fileDescriptorForSourceFiles, + icContext + ) private inner class InlineFunctionsMap( storageFile: File, @@ -628,9 +595,6 @@ open class IncrementalJvmCache( } } -private object PathCollectionExternalizer : - CollectionExternalizerV2>(PathStringDescriptor, { THashSet(CollectionFactory.createFilePathSet()) }) - sealed class ChangeInfo(val fqName: FqName) { open class MembersChanged(fqName: FqName, val names: Collection) : ChangeInfo(fqName) { override fun toStringProperties(): String = super.toStringProperties() + ", names = $names" diff --git a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt index 74dbdb9da2a..122288a9244 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt @@ -96,7 +96,7 @@ open class LookupStorage( val filtered = mutableSetOf() for (fileId in fileIds) { - val path = idToFile.getFile(fileId)?.path + val path = idToFile[fileId]?.path if (path != null) { paths.add(path) @@ -171,7 +171,7 @@ open class LookupStorage( lookupMap[hash] = lookupMap[hash]!!.filter { it in idToFile }.toSet() } - val oldFileToId = fileToId.toMap() + val oldFileToId = fileToId.keys.associateWith { fileToId[it]!! } val oldIdToNewId = HashMap(oldFileToId.size) idToFile.clear() fileToId.clear() @@ -302,17 +302,17 @@ private class TrackedLookupMap(private val lookupMap: LookupMap, private val tra val addedKeys = if (trackChanges) mutableSetOf() else null val removedKeys = if (trackChanges) mutableSetOf() else null - val keys: Collection + val keys: Set get() = lookupMap.keys - operator fun get(key: LookupSymbolKey): Collection? = lookupMap[key] + operator fun get(key: LookupSymbolKey): Set? = lookupMap[key] operator fun set(key: LookupSymbolKey, fileIds: Set) { recordSet(key) lookupMap[key] = fileIds } - fun append(key: LookupSymbolKey, fileIds: Collection) { + fun append(key: LookupSymbolKey, fileIds: Set) { recordSet(key) lookupMap.append(key, fileIds) } @@ -324,23 +324,19 @@ private class TrackedLookupMap(private val lookupMap: LookupMap, private val tra private fun recordSet(key: LookupSymbolKey) { if (!trackChanges) return - if (lookupMap[key] == null) { - if (key in removedKeys!!) { - removedKeys.remove(key) - } else { - addedKeys!!.add(key) - } + when (key) { + in lookupMap -> Unit + in removedKeys!! -> removedKeys.remove(key) + else -> addedKeys!!.add(key) } } private fun recordRemove(key: LookupSymbolKey) { if (!trackChanges) return - if (lookupMap[key] != null) { - if (key in addedKeys!!) { - addedKeys.remove(key) - } else { - removedKeys!!.add(key) - } + when (key) { + !in lookupMap -> Unit + in addedKeys!! -> addedKeys.remove(key) + else -> removedKeys!!.add(key) } } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt index 66913aef9e0..ebaa105187c 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt @@ -17,7 +17,6 @@ package org.jetbrains.kotlin.incremental.storage import com.intellij.util.io.DataExternalizer -import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.IOUtil import com.intellij.util.io.KeyDescriptor import org.jetbrains.annotations.TestOnly @@ -30,7 +29,9 @@ interface BasicMap : PersistentStorage { /** Removes all entries. */ fun clear() { - keys.forEach { remove(it) } + synchronized(this) { + keys.forEach { remove(it) } + } } /** @@ -40,22 +41,29 @@ interface BasicMap : PersistentStorage { * Make sure the storage has been closed first before calling this method. */ fun deleteStorageFiles() { - check(IOUtil.deleteAllFilesStartingWith(storageFile)) { - "Unable to delete storage file(s) with name prefix: ${storageFile.path}" + synchronized(this) { + check(IOUtil.deleteAllFilesStartingWith(storageFile)) { + "Unable to delete storage file(s) with name prefix: ${storageFile.path}" + } } } @TestOnly fun dump(): String { + val map: Map = synchronized(this) { + keys.associateWith { this[it]!! } + } + val printableMap: Map = map.map { + dumpKey(it.key) to dumpValue(it.value) + }.sortedBy { it.first }.toMap() + return StringBuilder().apply { Printer(this).apply { println("${storageFile.name.substringBefore(".tab")} (${this@BasicMap::class.java.simpleName})") pushIndent() - - for (key in keys.sortedBy { dumpKey(it) }) { - println("${dumpKey(key)} -> ${dumpValue(this@BasicMap[key]!!)}") + printableMap.forEach { + println("${it.key} -> ${it.value}") } - popIndent() } }.toString() @@ -65,59 +73,113 @@ interface BasicMap : PersistentStorage { fun dumpKey(key: KEY): String = key.toString() @TestOnly - fun dumpValue(value: VALUE): String = value.toString() + fun dumpValue(value: VALUE): String { + return if (value is Collection<*>) { + // Sort the elements so that we can reliably compare `Collection`s in tests (in case the order of the elements is not stable). + value.sortedBy { it.toString() }.toString() + } else { + value.toString() + } + } } abstract class BasicMapBase( - protected val storage: PersistentStorage, -) : PersistentStorageWrapper(storage), BasicMap + // TODO(KT-63456): Remove `protected val` (currently this property is still used by BasicStringMap) + protected val persistentStorage: PersistentStorage, +) : PersistentStorageWrapper(persistentStorage), BasicMap -abstract class AppendableBasicMapBase>( - protected val storage: AppendablePersistentStorage, -) : AppendablePersistentStorageWrapper(storage), BasicMap +abstract class AppendableBasicMapBase( + storage: AppendablePersistentStorage, +) : AppendablePersistentStorageWrapper(storage), BasicMap> abstract class AbstractBasicMap( storageFile: File, keyDescriptor: KeyDescriptor, valueExternalizer: DataExternalizer, - protected val icContext: IncrementalCompilationContext, + icContext: IncrementalCompilationContext, ) : BasicMapBase( createPersistentStorage(storageFile, keyDescriptor, valueExternalizer, icContext) -) { - protected val pathConverter - get() = icContext.pathConverter -} +) -abstract class AppendableAbstractBasicMap>( +abstract class AppendableBasicMap( storageFile: File, keyDescriptor: KeyDescriptor, elementExternalizer: DataExternalizer, - protected val icContext: IncrementalCompilationContext, -) : AppendableBasicMapBase( + icContext: IncrementalCompilationContext, +) : AppendableBasicMapBase( createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext) -) { - protected val pathConverter - get() = icContext.pathConverter +) + +abstract class AppendableSetBasicMap( + storageFile: File, + keyDescriptor: KeyDescriptor, + elementExternalizer: DataExternalizer, + icContext: IncrementalCompilationContext, +) : PersistentStorage>, BasicMap> { + + private val storage = createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext) + + override val storageFile: File = storage.storageFile + + @get:Synchronized + override val keys: Set + get() = storage.keys + + @Synchronized + override fun contains(key: KEY): Boolean = + storage.contains(key) + + @Synchronized + override fun get(key: KEY): Set? { + // Note: To optimize this getter, consider changing the type of `storage` from PersistentStorage> to + // PersistentStorage> so that we don't have to call `toSet()`. The downside is that it will make the code more complex, + // so we'll do it only if it's necessary. + return storage[key]?.toSet() + } + + @Synchronized + override fun set(key: KEY, value: Set) { + storage[key] = value + } + + @Synchronized + override fun remove(key: KEY) { + storage.remove(key) + } + + @Synchronized + override fun flush() { + storage.flush() + } + + @Synchronized + override fun close() { + storage.close() + } + + @Synchronized + fun append(key: KEY, element: E) { + storage.append(key, setOf(element)) + } + + @Synchronized + fun append(key: KEY, elements: Set) { + storage.append(key, elements) + } } abstract class BasicStringMap( storageFile: File, - keyDescriptor: KeyDescriptor, valueExternalizer: DataExternalizer, icContext: IncrementalCompilationContext, -) : AbstractBasicMap(storageFile, keyDescriptor, valueExternalizer, icContext) { - - constructor(storageFile: File, valueExternalizer: DataExternalizer, icContext: IncrementalCompilationContext) : - this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext) -} - -abstract class AppendableBasicStringMap>( - storageFile: File, - keyDescriptor: KeyDescriptor, - elementExternalizer: DataExternalizer, - icContext: IncrementalCompilationContext, -) : AppendableAbstractBasicMap(storageFile, keyDescriptor, elementExternalizer, icContext) { - - constructor(storageFile: File, elementExternalizer: DataExternalizer, icContext: IncrementalCompilationContext) : - this(storageFile, EnumeratorStringDescriptor.INSTANCE, elementExternalizer, icContext) +) : AbstractBasicMap( + storageFile, + StringExternalizer.toDescriptor(), + valueExternalizer, + icContext +) { + // TODO(KT-63456): Remove this property + // (To do this, we need to refactor all subclasses of BasicStringMap such that they don't use this property. For examples of what the + // outcome of the refactoring should look like, see other subclasses of AbstractBasicMap.) + protected val storage: PersistentStorage = persistentStorage } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt index adba75ae10b..eaf0bf0871d 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt @@ -16,49 +16,38 @@ package org.jetbrains.kotlin.incremental.storage +import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.dumpCollection import org.jetbrains.kotlin.name.FqName +import java.io.DataInput +import java.io.DataOutput import java.io.File internal open class ClassOneToManyMap( storageFile: File, icContext: IncrementalCompilationContext, -) : AppendableBasicStringMap>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { - override fun dumpValue(value: Collection): String = value.toSet().dumpCollection() +) : AppendableSetBasicMap( + storageFile, + LegacyFqNameExternalizer.toDescriptor(), + LegacyFqNameExternalizer, + icContext +) { @Synchronized - fun add(key: FqName, value: FqName) { - storage.append(key.asString(), value.asString()) - } - - @Synchronized - operator fun get(key: FqName): Collection = - storage[key.asString()]?.mapTo(mutableSetOf(), ::FqName) ?: setOf() - - @Synchronized - operator fun set(key: FqName, values: Collection) { - if (values.isEmpty()) { + override operator fun set(key: FqName, value: Set) { + if (value.isNotEmpty()) { + super.set(key, value) + } else { remove(key) - return } - - storage[key.asString()] = values.map(FqName::asString) } - @Synchronized - fun remove(key: FqName) { - storage.remove(key.asString()) - } - - // Access to caches could be done from multiple threads (e.g. JPS worker and RMI). The underlying collection is already synchronized, - // thus we need synchronization of this method and all modification methods. @Synchronized fun removeValues(key: FqName, removed: Set) { - val notRemoved = this[key].filter { it !in removed } - this[key] = notRemoved + this[key] = this[key].orEmpty() - removed } + } internal class SubtypesMap( @@ -70,3 +59,22 @@ internal class SupertypesMap( storageFile: File, icContext: IncrementalCompilationContext, ) : ClassOneToManyMap(storageFile, icContext) + +/** + * Use [LegacyFqNameExternalizer] instead of [FqNameExternalizer] for [SubtypesMap] for now because they internally use different types of + * `DataExternalizer`, and the `compiler-reference-index` module in the Kotlin IDEA plugin currently can only read the old data + * format (see KTIJ-27258). + * + * Once we fix that bug, we can remove this class and use [FqNameExternalizer]. + */ +private object LegacyFqNameExternalizer : DataExternalizer { + + override fun save(output: DataOutput, fqName: FqName) { + EnumeratorStringDescriptor.INSTANCE.save(output, fqName.asString()) + } + + override fun read(input: DataInput): FqName { + return FqName(EnumeratorStringDescriptor.INSTANCE.read(input)) + } + +} diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt index 33057321049..71f846d7089 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt @@ -5,33 +5,15 @@ package org.jetbrains.kotlin.incremental.storage -import com.intellij.util.io.EnumeratorStringDescriptor import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.dumpCollection import java.io.File class ComplementarySourceFilesMap( storageFile: File, icContext: IncrementalCompilationContext, -) : AppendableBasicStringMap>( +) : AppendableSetBasicMap( storageFile, - PathStringDescriptor, - EnumeratorStringDescriptor.INSTANCE, + icContext.fileDescriptorForSourceFiles, + icContext.fileDescriptorForSourceFiles, icContext -) { - - operator fun set(sourceFile: File, complementaryFiles: Collection) { - storage[pathConverter.toPath(sourceFile)] = pathConverter.toPaths(complementaryFiles) - } - - operator fun get(sourceFile: File): Collection { - val paths = storage[pathConverter.toPath(sourceFile)].orEmpty() - return pathConverter.toFiles(paths).toSet() - } - - override fun dumpValue(value: Collection) = - value.dumpCollection() - - fun remove(file: File): Collection = - get(file).also { storage.remove(pathConverter.toPath(file)) } -} \ No newline at end of file +) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt index 4352a1b9aed..ad3d83725d0 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToIdMap.kt @@ -20,27 +20,11 @@ import org.jetbrains.kotlin.incremental.IncrementalCompilationContext import java.io.File internal class FileToIdMap( - file: File, + storageFile: File, icContext: IncrementalCompilationContext, -) : BasicStringMap(file, IntExternalizer, icContext) { - override fun dumpValue(value: Int): String = value.toString() - - operator fun get(file: File): Int? = storage[pathConverter.toPath(file)] - - operator fun set(file: File, id: Int) { - storage[pathConverter.toPath(file)] = id - } - - fun remove(file: File) { - storage.remove(pathConverter.toPath(file)) - } - - fun toMap(): Map { - val result = HashMap() - for (key in storage.keys) { - val value = storage[key] ?: continue - result[pathConverter.toFile(key)] = value - } - return result - } -} +) : AbstractBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + IntExternalizer, + icContext +) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt index d32708ca940..bb2eeabf125 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/FileToPathConverter.kt @@ -5,22 +5,54 @@ package org.jetbrains.kotlin.incremental.storage +import com.intellij.util.io.KeyDescriptor +import java.io.DataInput +import java.io.DataOutput import java.io.File -/** Converts a [File] to a path of type [String] to store in IC caches, and vice versa. */ +/** Converts a [File] to a path of type [String] (and vice versa) for serialization of file paths in IC caches. */ interface FileToPathConverter { fun toPath(file: File): String fun toFile(path: String): File + fun getFileDescriptor(): KeyDescriptor = FileDescriptor(this) } -fun FileToPathConverter.toPaths(files: Collection): List = - files.map { toPath(it) } - -fun FileToPathConverter.toFiles(paths: Collection): List = - paths.map { toFile(it) } - -object FileToAbsolutePathConverter : FileToPathConverter { - override fun toPath(file: File): String = file.normalize().absolutePath - +object BasicFileToPathConverter : FileToPathConverter { + override fun toPath(file: File): String = file.path override fun toFile(path: String): File = File(path) } + +/** + * [KeyDescriptor] for a [File] which uses the given [pathConverter] for serialization of file paths. + * + * To support build cache relocatability, the user of this class can provide a [RelocatableFileToPathConverter] as the [pathConverter]. + */ +private class FileDescriptor(private val pathConverter: FileToPathConverter) : KeyDescriptor { + + override fun save(output: DataOutput, file: File) { + StringExternalizer.save(output, pathConverter.toPath(file)) + } + + override fun read(input: DataInput): File { + return pathConverter.toFile(StringExternalizer.read(input)) + } + + override fun getHashCode(file: File): Int { + // It's important to use the pathConverter to get a possibly relocatable path first before computing the hash code because the + // returned hash code affects the contents of IC caches. + // For example, if we get the hash code directly without using pathConverter, the hash codes of the following files will be + // different and therefore the contents of the IC caches will also be different (i.e., the IC caches will not be relocatable): + // File("/path/to/project1/src/foo.kt").hashCode() = 123 + // File("/path/to/project2/src/foo.kt").hashCode() = 456 + // If we use a RelocatableFileToPathConverter instead, the file paths will be normalized first, so we'll get the same hash codes: + // [In "/path/to/project1"] File("src/foo.kt").hashCode() = 789 + // [In "/path/to/project2"] File("src/foo.kt").hashCode() = 789 + // It's also important to get the hash code using `File.hashCode()` instead of `String.hashCode()` because the hashCode() method + // needs to account for case-insensitive filesystems. + return File(pathConverter.toPath(file)).hashCode() + } + + override fun isEqual(file1: File, file2: File): Boolean { + return file1 == file2 + } +} diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt index eb50dbedb02..45af936b661 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt @@ -16,23 +16,42 @@ package org.jetbrains.kotlin.incremental.storage +import com.intellij.util.io.DataExternalizer import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.ExternalIntegerKeyDescriptor import org.jetbrains.kotlin.incremental.IncrementalCompilationContext +import java.io.DataInput +import java.io.DataOutput import java.io.File internal class IdToFileMap( - file: File, + storageFile: File, icContext: IncrementalCompilationContext, -) : AbstractBasicMap(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE, icContext) { - override fun dumpKey(key: Int): String = key.toString() +) : AbstractBasicMap( + storageFile, + LegacyIntExternalizer.toDescriptor(), + LegacyFileExternalizer(icContext.pathConverterForSourceFiles), + icContext +) - override fun dumpValue(value: String): String = value +/** Similar to [LegacyFileExternalizer]. */ +private val LegacyIntExternalizer = ExternalIntegerKeyDescriptor.INSTANCE - fun getFile(id: Int): File? = storage[id]?.let { pathConverter.toFile(it) } +/** + * Use [LegacyFqNameExternalizer] instead of [org.jetbrains.kotlin.incremental.IncrementalCompilationContext.fileDescriptorForOutputFiles] + * for [IdToFileMap] for now because they internally use different types of `DataExternalizer`, and the `compiler-reference-index` + * module in the Kotlin IDEA plugin currently can only read the old data format (see KTIJ-27258). + * + * Once we fix that bug, we can remove this class and use + * [org.jetbrains.kotlin.incremental.IncrementalCompilationContext.fileDescriptorForOutputFiles]. + */ +private class LegacyFileExternalizer(private val pathConverter: FileToPathConverter) : DataExternalizer { - operator fun set(id: Int, file: File) { - storage[id] = pathConverter.toPath(file) + override fun save(output: DataOutput, file: File) { + EnumeratorStringDescriptor.INSTANCE.save(output, pathConverter.toPath(file)) } + override fun read(input: DataInput): File { + return pathConverter.toFile(EnumeratorStringDescriptor.INSTANCE.read(input)) + } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt index c21b10d35c8..0af8a0cbef3 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt @@ -59,14 +59,14 @@ open class InMemoryStorage( } @Synchronized - override fun get(key: KEY): VALUE? = addedEntries[key] ?: modifiedEntries[key] ?: when (key) { - in appendedEntries -> { - @Suppress("UNCHECKED_CAST") - ((storage[key]!! as Collection<*>) + (appendedEntries[key]!! as Collection<*>)) as VALUE - } - in removedKeys -> null - else -> storage[key] - } + override fun get(key: KEY): VALUE? = + addedEntries[key] + ?: modifiedEntries[key] + // appendedEntries is handled by AppendableInMemoryStorage#get + ?: when (key) { + in removedKeys -> null + else -> storage[key] + } @Synchronized override fun set(key: KEY, value: VALUE) = when (key) { @@ -110,7 +110,7 @@ open class InMemoryStorage( modifiedEntries.forEach { storage[it.key] = it.value } - check(appendedEntries.isEmpty()) { "appendedEntries is not empty" } + // appendedEntries is handled by AppendableInMemoryStorage#applyChanges removedKeys.forEach { storage.remove(it) } @@ -139,18 +139,23 @@ open class InMemoryStorage( } -/** [InMemoryStorage] where a map entry's value is a [Collection]. */ +/** [InMemoryStorage] where a map entry's value is a [Collection] of elements of type [E]. */ @ThreadSafe -class AppendableInMemoryStorage>( - private val storage: AppendablePersistentStorage, -) : InMemoryStorage(storage), AppendablePersistentStorage { +class AppendableInMemoryStorage( + private val storage: AppendablePersistentStorage, +) : InMemoryStorage>(storage), AppendablePersistentStorage { - @Suppress("UNCHECKED_CAST") @Synchronized - override fun append(key: KEY, elements: VALUE) = when (key) { - in addedEntries -> addedEntries[key] = (addedEntries[key]!! + elements) as VALUE - in modifiedEntries -> modifiedEntries[key] = (modifiedEntries[key]!! + elements) as VALUE - in appendedEntries -> appendedEntries[key] = (appendedEntries[key]!! + elements) as VALUE + override fun get(key: KEY): Collection? = when (key) { + in appendedEntries -> storage[key]!! + appendedEntries[key]!! + else -> super.get(key) + } + + @Synchronized + override fun append(key: KEY, elements: Collection) = when (key) { + in addedEntries -> addedEntries[key] = addedEntries[key]!! + elements + in modifiedEntries -> modifiedEntries[key] = modifiedEntries[key]!! + elements + in appendedEntries -> appendedEntries[key] = appendedEntries[key]!! + elements in removedKeys -> { removedKeys.remove(key) // Note: We update `modifiedEntries` (not `appendedEntries`), because if the entry is removed and then appended, it is diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt index 08d932fb2ee..9a55f371762 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt @@ -95,19 +95,19 @@ open class LazyStorage( } -/** [LazyStorage] where a map entry's value is a [Collection]. */ +/** [LazyStorage] where a map entry's value is a [Collection] of elements of type [E]. */ @ThreadSafe -class AppendableLazyStorage>( +class AppendableLazyStorage( storageFile: File, keyDescriptor: KeyDescriptor, elementExternalizer: DataExternalizer, -) : LazyStorage(storageFile, keyDescriptor, AppendableCollectionExternalizer(elementExternalizer)), - AppendablePersistentStorage { +) : LazyStorage>(storageFile, keyDescriptor, AppendableCollectionExternalizer(elementExternalizer)), + AppendablePersistentStorage { private val appendableCollectionExternalizer = AppendableCollectionExternalizer(elementExternalizer) @Synchronized - override fun append(key: KEY, elements: VALUE) { + override fun append(key: KEY, elements: Collection) { getStorageOrCreateNew().appendData( key, AppendablePersistentMap.ValueDataAppender { appendableCollectionExternalizer.append(it, elements) } @@ -116,25 +116,25 @@ class AppendableLazyStorage>( } /** - * [DataExternalizer] for a [Collection] of type [E]. + * [DataExternalizer] for a [Collection] of elements of type [E]. * * IMPORTANT: It is a *private* class because it is meant to be used only with a [PersistentHashMap] (e.g., the [read] method reads until * the stream ends, [append] can be called multiple times and its implementation is identical to [save] -- these only work with a * [PersistentHashMap]). */ -private class AppendableCollectionExternalizer>( +private class AppendableCollectionExternalizer( private val elementExternalizer: DataExternalizer, -) : DataExternalizer { +) : DataExternalizer> { - fun append(output: DataOutput, elements: VALUE) { + fun append(output: DataOutput, elements: Collection) { save(output, elements) } - override fun save(output: DataOutput, value: VALUE) { + override fun save(output: DataOutput, value: Collection) { value.forEach { elementExternalizer.save(output, it) } } - override fun read(input: DataInput): VALUE { + override fun read(input: DataInput): Collection { val result = ArrayList() val stream = input as DataInputStream @@ -142,8 +142,7 @@ private class AppendableCollectionExternalizer>( result.add(elementExternalizer.read(stream)) } - @Suppress("UNCHECKED_CAST") - return result as VALUE + return result } } @@ -164,13 +163,13 @@ fun createPersistentStorage( } } -fun > createAppendablePersistentStorage( +fun createAppendablePersistentStorage( storageFile: File, keyDescriptor: KeyDescriptor, elementExternalizer: DataExternalizer, icContext: IncrementalCompilationContext, -): AppendablePersistentStorage { - return AppendableLazyStorage(storageFile, keyDescriptor, elementExternalizer).let { storage -> +): AppendablePersistentStorage { + return AppendableLazyStorage(storageFile, keyDescriptor, elementExternalizer).let { storage -> if (icContext.keepIncrementalCompilationCachesInMemory) { AppendableInMemoryStorage(storage).also { icContext.transaction.registerInMemoryStorageWrapper(it) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt index 0d11976e7fb..7bc6cfdeb0a 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt @@ -22,26 +22,15 @@ import java.io.File class LookupMap( storage: File, icContext: IncrementalCompilationContext, -) : - AppendableAbstractBasicMap>( - storage, - LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache), - IntExternalizer, - icContext, - ) { - - override fun dumpKey(key: LookupSymbolKey): String = key.toString() - - override fun dumpValue(value: Collection): String = value.toString() +) : AppendableSetBasicMap( + storage, + LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache), + IntExternalizer, + icContext, +) { + @Synchronized fun add(name: String, scope: String, fileId: Int) { - storage.append(LookupSymbolKey(name, scope), fileId) + append(LookupSymbolKey(name, scope), fileId) } - - override fun get(key: LookupSymbolKey): Collection? = storage[key]?.toSet() - - operator fun set(key: LookupSymbolKey, fileIds: Set) { - storage[key] = fileIds - } - } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt index b18061d93b1..ab397a6d85d 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt @@ -50,16 +50,15 @@ interface PersistentStorage : Closeable { override fun close() } -/** [PersistentStorage] where a map entry's value is a [Collection]. */ -interface AppendablePersistentStorage> : PersistentStorage { +/** [PersistentStorage] where a map entry's value is a [Collection] of elements of type [E]. */ +interface AppendablePersistentStorage : PersistentStorage> { /** Adds the given [elements] to the collection corresponding to the given [key]. */ - fun append(key: KEY, elements: VALUE) + fun append(key: KEY, elements: Collection) /** Adds the given [element] to the collection corresponding to the given [key]. */ fun append(key: KEY, element: E) { - @Suppress("UNCHECKED_CAST") - append(key, listOf(element) as VALUE) + append(key, listOf(element)) } } @@ -111,14 +110,14 @@ abstract class PersistentStorageWrapper( } } -/** [PersistentStorageWrapper] where a map entry's value is a [Collection]. */ +/** [PersistentStorageWrapper] where a map entry's value is a [Collection] of elements of type [E]. */ @ThreadSafe -abstract class AppendablePersistentStorageWrapper>( - private val appendableStorage: AppendablePersistentStorage, -) : PersistentStorageWrapper(appendableStorage), AppendablePersistentStorage { +abstract class AppendablePersistentStorageWrapper( + private val appendableStorage: AppendablePersistentStorage, +) : PersistentStorageWrapper>(appendableStorage), AppendablePersistentStorage { @Synchronized - override fun append(key: KEY, elements: VALUE) { + override fun append(key: KEY, elements: Collection) { appendableStorage.append(key, elements) } } diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt deleted file mode 100644 index 3c3a68c6a7f..00000000000 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.incremental.storage - -import org.jetbrains.kotlin.utils.ThreadSafe -import java.io.File - -/** - * [PersistentStorage] which maps [KEY] to [VALUE] as viewed by the users of this class, but delegates operations to another [storage] which - * maps [INTERNAL_KEY] to [INTERNAL_VALUE]. - * - * The users of this class need to provide the transformations from [KEY] to [INTERNAL_KEY], [VALUE] to [INTERNAL_VALUE], and vice versa. - */ -@ThreadSafe -abstract class PersistentStorageAdapter( - private val storage: PersistentStorage, - private val publicToInternalKey: (KEY) -> INTERNAL_KEY, - private val internalToPublicKey: (INTERNAL_KEY) -> KEY, - private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE, - private val internalToPublicValue: (INTERNAL_VALUE) -> VALUE, -) : PersistentStorage, BasicMap { - - override val storageFile: File = storage.storageFile - - @get:Synchronized - override val keys: Set - get() = storage.keys.mapTo(LinkedHashSet()) { internalToPublicKey(it) } - - @Synchronized - override fun contains(key: KEY): Boolean = - storage.contains(publicToInternalKey(key)) - - @Synchronized - override fun get(key: KEY): VALUE? = - storage[publicToInternalKey(key)]?.let { internalToPublicValue(it) } - - @Synchronized - override fun set(key: KEY, value: VALUE) { - storage[publicToInternalKey(key)] = publicToInternalValue(value) - } - - @Synchronized - override fun remove(key: KEY) { - storage.remove(publicToInternalKey(key)) - } - - @Synchronized - override fun flush() { - storage.close() - } - - @Synchronized - override fun close() { - storage.close() - } -} - -/** [PersistentStorageAdapter] where a map entry's value is a [Collection]. */ -@ThreadSafe -abstract class AppendablePersistentStorageAdapter, INTERNAL_KEY, INTERNAL_E, INTERNAL_VALUE : Collection>( - private val storage: AppendablePersistentStorage, - private val publicToInternalKey: (KEY) -> INTERNAL_KEY, - internalToPublicKey: (INTERNAL_KEY) -> KEY, - private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE, - internalToPublicValue: (INTERNAL_VALUE) -> VALUE, -) : PersistentStorageAdapter( - storage, - publicToInternalKey, - internalToPublicKey, - publicToInternalValue, - internalToPublicValue -), AppendablePersistentStorage { - - @Synchronized - override fun append(key: KEY, elements: VALUE) { - storage.append(publicToInternalKey(key), publicToInternalValue(elements)) - } -} diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt index 80534caca45..8a66cecc27e 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/RelocatableFileToPathConverter.kt @@ -17,6 +17,8 @@ import java.io.File class RelocatableFileToPathConverter(private val baseDir: File) : FileToPathConverter { override fun toPath(file: File): String { + check(file.isAbsolute) { "Expected absolute path but found relative path: ${file.path}" } + // Note: If the given file is located outside `baseDir`, the relative path will start with "../". It's not "clean", but it can work. // TODO: Re-design the code such that `baseDir` always contains the given file (also add a precondition check here). return file.relativeTo(baseDir).invariantSeparatorsPath diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt index 167231a265a..375d3d54499 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt @@ -5,44 +5,15 @@ package org.jetbrains.kotlin.incremental.storage -import com.intellij.util.io.EnumeratorStringDescriptor import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.dumpCollection import java.io.File class SourceToJsOutputMap( storageFile: File, icContext: IncrementalCompilationContext, -) : AppendableBasicStringMap>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) { - override fun dumpValue(value: Collection): String = value.dumpCollection() - - @Synchronized - fun add(key: File, value: File) { - storage.append(pathConverter.toPath(key), pathConverter.toPath(value)) - } - - operator fun get(sourceFile: File): Collection = - storage[pathConverter.toPath(sourceFile)]?.mapTo(mutableSetOf()) { pathConverter.toFile(it) } ?: setOf() - - - @Synchronized - operator fun set(key: File, values: Collection) { - if (values.isEmpty()) { - remove(key) - return - } - - storage[pathConverter.toPath(key)] = values.toSet().map { pathConverter.toPath(it) } - } - - @Synchronized - fun remove(key: File) { - storage.remove(pathConverter.toPath(key)) - } - - @Synchronized - fun removeValues(key: File, removed: Set) { - val notRemoved = this[key].filter { it !in removed } - this[key] = notRemoved - } -} \ No newline at end of file +) : AppendableSetBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + icContext.fileDescriptorForOutputFiles, + icContext +) diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt index a460a1cbc8e..b8134fa9785 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt @@ -16,11 +16,12 @@ package org.jetbrains.kotlin.incremental.storage -import com.intellij.util.io.EnumeratorStringDescriptor +import com.intellij.util.io.DataExternalizer import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.dumpCollection import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName +import java.io.DataInput +import java.io.DataOutput import java.io.File internal class SourceToJvmNameMap( @@ -37,30 +38,27 @@ internal abstract class AbstractSourceToOutputMap( private val nameTransformer: NameTransformer, storageFile: File, icContext: IncrementalCompilationContext, -) : AppendableBasicStringMap>( +) : AppendableSetBasicMap( storageFile, - PathStringDescriptor, - EnumeratorStringDescriptor.INSTANCE, + icContext.fileDescriptorForSourceFiles, + NameExternalizer(nameTransformer), icContext ) { - fun clearOutputsForSource(sourceFile: File) { - remove(pathConverter.toPath(sourceFile)) + + @Synchronized + fun getFqNames(sourceFile: File): Collection? = + this[sourceFile]?.map { nameTransformer.asFqName(nameTransformer.asString(it)) } + +} + +private class NameExternalizer(private val nameTransformer: NameTransformer) : DataExternalizer { + + override fun save(output: DataOutput, name: Name) { + StringExternalizer.save(output, nameTransformer.asString(name)) } - fun add(sourceFile: File, className: Name) { - storage.append(pathConverter.toPath(sourceFile), nameTransformer.asString(className)) + override fun read(input: DataInput): Name { + return nameTransformer.asName(StringExternalizer.read(input)) } - fun contains(sourceFile: File): Boolean = - pathConverter.toPath(sourceFile) in storage - - operator fun get(sourceFile: File): Collection = - storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asName) - - fun getFqNames(sourceFile: File): Collection = - storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asFqName) - - override fun dumpValue(value: Collection) = - value.dumpCollection() - } \ No newline at end of file diff --git a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt index 5e8905a11f0..0f78418b5bb 100644 --- a/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt +++ b/build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt @@ -16,10 +16,8 @@ package org.jetbrains.kotlin.incremental.storage -import com.intellij.openapi.util.io.FileUtil -import com.intellij.openapi.util.text.StringUtil +import com.intellij.util.containers.hash.EqualityPolicy import com.intellij.util.io.DataExternalizer -import com.intellij.util.io.EnumeratorStringDescriptor import com.intellij.util.io.IOUtil import com.intellij.util.io.KeyDescriptor import org.jetbrains.kotlin.inline.InlineFunction @@ -31,10 +29,21 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.resolve.jvm.JvmClassName import java.io.* +class DefaultEqualityPolicy : EqualityPolicy { + override fun getHashCode(value: T): Int = value.hashCode() + override fun isEqual(value1: T, value2: T): Boolean = (value1 == value2) +} + +fun DataExternalizer.toDescriptor(): KeyDescriptor = + object : KeyDescriptor, + DataExternalizer by this, + EqualityPolicy by DefaultEqualityPolicy() { + } + class LookupSymbolKeyDescriptor( /** If `true`, original values are saved; if `false`, only hashes are saved. */ private val storeFullFqNames: Boolean = false -) : KeyDescriptor { +) : KeyDescriptor, EqualityPolicy by DefaultEqualityPolicy() { override fun read(input: DataInput): LookupSymbolKey { // Note: The value of the storeFullFqNames variable below may or may not be the same as LookupSymbolKeyDescriptor.storeFullFqNames. @@ -67,10 +76,6 @@ class LookupSymbolKeyDescriptor( output.writeInt(value.scopeHash) } } - - override fun getHashCode(value: LookupSymbolKey): Int = value.hashCode() - - override fun isEqual(val1: LookupSymbolKey, val2: LookupSymbolKey): Boolean = val1 == val2 } object FqNameExternalizer : DataExternalizer { @@ -220,21 +225,6 @@ object StringExternalizer : DataExternalizer { override fun read(input: DataInput): String = IOUtil.readString(input) } -object PathStringDescriptor : EnumeratorStringDescriptor() { - override fun getHashCode(path: String): Int { - return if (StringUtil.isEmpty(path)) 0 else FileUtil.toCanonicalPath(path).hashCode() - } - - override fun isEqual(val1: String, val2: String?): Boolean { - if (val1 == val2) return true - if (val2 == null) return false - - val path1 = FileUtil.toCanonicalPath(val1) - val path2 = FileUtil.toCanonicalPath(val2) - return path1 == path2 - } -} - /** [DataExternalizer] that delegates to another [DataExternalizer] depending on the type of the object to externalize. */ class DelegateDataExternalizer( val types: List>, @@ -328,7 +318,7 @@ private class CollectionExternalizerForPersistentHashMap( * (see [CollectionExternalizerForPersistentHashMap]). * * Currently, we can't change the name or implementation of this class because it is still used by the `compiler-reference-index` module in - * the Kotlin IDEA plugin and that code relies on this name and implementation being unchanged (see KT-62288). + * the Kotlin IDEA plugin and that code relies on this name and implementation being unchanged (see KTIJ-27258). * * Once we remove that dependency, we can remove this class. */ @@ -345,7 +335,7 @@ object IntCollectionExternalizer : /** [DataExternalizer] for a [Collection]. */ open class CollectionExternalizerV2>( private val elementExternalizer: DataExternalizer, - private val newCollection: (size: Int) -> MutableCollection, + private val newCollection: (size: Int) -> MutableCollection = { size -> ArrayList(size) }, ) : DataExternalizer { override fun save(output: DataOutput, collection: C) { @@ -369,8 +359,6 @@ open class CollectionExternalizerV2>( } } -object StringCollectionExternalizer : CollectionExternalizerV2>(StringExternalizer, { size -> ArrayList(size) }) - class ListExternalizer(elementExternalizer: DataExternalizer) : CollectionExternalizerV2>(elementExternalizer, { size -> ArrayList(size) }) diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt index 5d4de8ed58f..df6fb50defd 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt @@ -50,20 +50,13 @@ abstract class IncrementalCachesManager) { - val sourceToOutput = MultiMap() + val sourceToOutput = MultiMap.createLinked() for (generatedFile in generatedFiles) { for (source in generatedFile.sourceFiles) { @@ -61,7 +61,7 @@ class InputsCache( } for ((source, outputs) in sourceToOutput.entrySet()) { - sourceToOutputMap[source] = outputs + sourceToOutputMap[source] = outputs.toSet() } } } \ No newline at end of file diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt index 6e7fc85b4a9..d3a96e6ed5d 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMap.kt @@ -18,41 +18,39 @@ package org.jetbrains.kotlin.incremental.snapshots import org.jetbrains.kotlin.incremental.ChangedFiles import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.storage.BasicStringMap -import org.jetbrains.kotlin.incremental.storage.PathStringDescriptor +import org.jetbrains.kotlin.incremental.storage.AbstractBasicMap import java.io.File -import java.util.* class FileSnapshotMap( storageFile: File, icContext: IncrementalCompilationContext, -) : BasicStringMap(storageFile, PathStringDescriptor, FileSnapshotExternalizer, icContext) { - - override fun dumpValue(value: FileSnapshot): String = - value.toString() - +) : AbstractBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + FileSnapshotExternalizer, + icContext +) { @Synchronized fun compareAndUpdate(newFiles: Iterable): ChangedFiles.Known { val snapshotProvider = SimpleFileSnapshotProviderImpl() val newOrModified = ArrayList() val removed = ArrayList() - val newPaths = newFiles.mapTo(HashSet(), transform = pathConverter::toPath) - for (oldPath in storage.keys) { - if (oldPath !in newPaths) { - storage.remove(oldPath) - removed.add(pathConverter.toFile(oldPath)) + val newFilesSet = newFiles.toSet() + for (oldFile in keys) { + if (oldFile !in newFilesSet) { + remove(oldFile) + removed.add(oldFile) } } - for (path in newPaths) { - val file = pathConverter.toFile(path) - val oldSnapshot = storage[path] + for (file in newFilesSet) { + val oldSnapshot = this[file] val newSnapshot = snapshotProvider[file] if (oldSnapshot == null || oldSnapshot != newSnapshot) { newOrModified.add(file) - storage[path] = newSnapshot + this[file] = newSnapshot } } diff --git a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt index f82d63529df..d38f59517fe 100644 --- a/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt +++ b/compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt @@ -6,24 +6,21 @@ package org.jetbrains.kotlin.incremental.storage import org.jetbrains.kotlin.incremental.IncrementalCompilationContext -import org.jetbrains.kotlin.incremental.dumpCollection import java.io.File class SourceToOutputFilesMap( storageFile: File, icContext: IncrementalCompilationContext, -) : BasicStringMap>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) { - operator fun set(sourceFile: File, outputFiles: Collection) { - storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] = - outputFiles.toSet().map(icContext.pathConverterForOutputFiles::toPath) +) : AppendableSetBasicMap( + storageFile, + icContext.fileDescriptorForSourceFiles, + icContext.fileDescriptorForOutputFiles, + icContext +) { + @Synchronized + fun getAndRemove(sourceFile: File): Set? { + return get(sourceFile).also { + remove(sourceFile) + } } - - operator fun get(sourceFile: File): Collection? = - storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)]?.map(icContext.pathConverterForOutputFiles::toFile) - - override fun dumpValue(value: Collection) = - value.dumpCollection() - - fun getAndRemove(file: File): Collection? = - get(file).also { storage.remove(icContext.pathConverterForSourceFiles.toPath(file)) } -} \ No newline at end of file +} diff --git a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt index e9001c15d8d..f33b9b0e1ca 100644 --- a/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt +++ b/compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt @@ -60,49 +60,53 @@ class SourceToOutputFilesMapTest { @Test fun testSetOneGetReturnsOne() { - stofMap[fooDotKt] = listOf(fooDotClass) + stofMap[fooDotKt] = setOf(fooDotClass) - assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) + assertEquals(setOf(fooDotClass), stofMap[fooDotKt]) } @Test fun testSetDupeGetReturnsUnique() { - stofMap[fooDotKt] = listOf(fooDotClass, fooDotClass) + stofMap.append(fooDotKt, fooDotClass) + stofMap.append(fooDotKt, fooDotClass) - assertEquals(listOf(fooDotClass), stofMap[fooDotKt]) + assertEquals(setOf(fooDotClass), stofMap[fooDotKt]) } @Test fun testSetOverwriteGetReturnsNew() { val fooKtDotClass = classesDir.resolve("FooKt.class") - stofMap[fooDotKt] = listOf(fooDotClass) - stofMap[fooDotKt] = listOf(fooKtDotClass) + stofMap[fooDotKt] = setOf(fooDotClass) + stofMap[fooDotKt] = setOf(fooKtDotClass) - assertEquals(listOf(fooKtDotClass), stofMap[fooDotKt]) + assertEquals(setOf(fooKtDotClass), stofMap[fooDotKt]) } @Test - fun testSetRelativeFails() { - assertFailsWith { - stofMap[fooDotKt] = listOf(File("relativePath")) + fun testSetRelativePathFails() { + assertFailsWith { + stofMap[fooDotKt] = setOf(File("relativePath")) + } + assertFailsWith { + stofMap[File("relativePath")] = setOf(fooDotClass) } } @Test - fun testGetRelativeFails() { - stofMap[fooDotKt] = listOf(fooDotClass) + fun testGetRelativePathFails() { + stofMap[fooDotKt] = setOf(fooDotClass) - assertFailsWith { - stofMap[fooDotKt.relativeTo(srcDir)] + assertFailsWith { + stofMap[File("relativePath")] } } @Test fun testGetAndRemove() { - stofMap[fooDotKt] = listOf(fooDotClass) + stofMap[fooDotKt] = setOf(fooDotClass) - assertEquals(listOf(fooDotClass), stofMap.getAndRemove(fooDotKt)) + assertEquals(setOf(fooDotClass), stofMap.getAndRemove(fooDotKt)) assertNull(stofMap[fooDotKt]) } -} \ No newline at end of file +} diff --git a/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt b/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt index 6eb02d17498..a13727eac44 100644 --- a/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt +++ b/repo/codebase-tests/tests/org/jetbrains/kotlin/code/CodeConformanceTest.kt @@ -218,7 +218,6 @@ class CodeConformanceTest : TestCase() { "These files are affected:\n%s", allowedFiles = listOf( "analysis/light-classes-base/src/org/jetbrains/kotlin/asJava/classes/KotlinClassInnerStuffCache.kt", - "build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/CliVirtualFileFinder.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCliJavaFileManagerImpl.kt", "compiler/cli/cli-base/src/org/jetbrains/kotlin/cli/jvm/index/JvmDependenciesIndexImpl.kt",