[IC] Refactor IC maps to reuse code and ensure consistency

The key changes include:
  - Top interface: `PersistentStorage`
  - Implementation:
      + `LazyStorage`
      + `InMemoryStorage`
  - Extended functionality:
      + `BasicMap`
      + `PersistentStorageAdapter`

 The remaining changes are small cleanups to adapt to the above key
 changes.

 Test: Existing tests (refactoring change)
 Bug: N/A (code cleanup)

build-common/src/org/jetbrains/kotlin/incremental/CompilationTransaction.kt

@@ -11,7 +11,7 @@ import org.jetbrains.kotlin.build.report.metrics.GradleBuildPerformanceMetric
 import org.jetbrains.kotlin.build.report.metrics.GradleBuildTime
 import org.jetbrains.kotlin.build.report.metrics.measure
 import org.jetbrains.kotlin.compilerRunner.OutputItemsCollector
-import org.jetbrains.kotlin.incremental.storage.InMemoryStorageWrapper
+import org.jetbrains.kotlin.incremental.storage.InMemoryStorageInterface
 import org.jetbrains.kotlin.konan.file.use
 import java.io.Closeable
 import java.io.File
@@ -48,7 +48,7 @@ interface CompilationTransaction : Closeable {
      */
     var executionThrowable: Throwable?
 
-    fun registerInMemoryStorageWrapper(inMemoryStorageWrapper: InMemoryStorageWrapper<*, *>)
+    fun registerInMemoryStorageWrapper(inMemoryStorageWrapper: InMemoryStorageInterface<*, *>)
 }
 
 fun CompilationTransaction.write(file: Path, writeAction: () -> Unit) {
@@ -86,7 +86,7 @@ inline fun <R> CompilationTransaction.runWithin(
 }
 
 abstract class BaseCompilationTransaction : CompilationTransaction {
-    private val inMemoryStorageWrappers = hashSetOf<InMemoryStorageWrapper<*, *>>()
+    private val inMemoryStorageWrappers = hashSetOf<InMemoryStorageInterface<*, *>>()
 
     protected var isSuccessful: Boolean = false
 
@@ -94,7 +94,7 @@ abstract class BaseCompilationTransaction : CompilationTransaction {
         isSuccessful = true
     }
 
-    override fun registerInMemoryStorageWrapper(inMemoryStorageWrapper: InMemoryStorageWrapper<*, *>) {
+    override fun registerInMemoryStorageWrapper(inMemoryStorageWrapper: InMemoryStorageInterface<*, *>) {
         inMemoryStorageWrappers.add(inMemoryStorageWrapper)
     }
 
@@ -117,7 +117,7 @@ abstract class BaseCompilationTransaction : CompilationTransaction {
     protected fun closeCachesManager() = runCatching {
         if (!isSuccessful) {
             for (wrapper in inMemoryStorageWrappers) {
-                wrapper.resetInMemoryChanges()
+                wrapper.clearChanges()
             }
         }
         cachesManager?.close()

build-common/src/org/jetbrains/kotlin/incremental/IncrementalCompilationContext.kt

@@ -11,7 +11,7 @@ import org.jetbrains.kotlin.incremental.storage.FileToAbsolutePathConverter
 import org.jetbrains.kotlin.incremental.storage.FileToPathConverter
 
 class IncrementalCompilationContext(
-    // The root directories of source files and class files are different, so we may need different `FileToPathConverter`s
+    // The root directories of source files and output files are different, so we may need different `FileToPathConverter`s
     val pathConverterForSourceFiles: FileToPathConverter = FileToAbsolutePathConverter,
     val pathConverterForOutputFiles: FileToPathConverter = FileToAbsolutePathConverter,
     val storeFullFqNamesInLookupCache: Boolean = false,
@@ -47,6 +47,6 @@ class IncrementalCompilationContext(
     )
 
     // FIXME: Remove `pathConverter` and require its users to decide whether to use `pathConverterForSourceFiles` or
-    // `pathConverterForClassFiles`
+    // `pathConverterForOutputFiles`
     val pathConverter = pathConverterForSourceFiles
-}
+}

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJsCache.kt

@@ -154,7 +154,7 @@ open class IncrementalJsCache(
         }
         removeAllFromClassStorage(dirtyOutputClassesMap.getDirtyOutputClasses(), changesCollector)
         dirtySources.clear()
-        dirtyOutputClassesMap.clean()
+        dirtyOutputClassesMap.clear()
     }
 
     fun nonDirtyPackageParts(): Map<File, TranslationResultValue> =
@@ -447,13 +447,7 @@ private class PackageMetadataMap(
         storage[packageName] = newMetadata
     }
 
-    fun remove(packageName: String) {
-        storage.remove(packageName)
-    }
-
     fun keys() = storage.keys
 
-    operator fun get(packageName: String) = storage[packageName]
-
     override fun dumpValue(value: ByteArray): String = "Package metadata: ${value.md5()}"
 }

build-common/src/org/jetbrains/kotlin/incremental/IncrementalJvmCache.kt

@@ -94,7 +94,7 @@ open class IncrementalJvmCache(
         sourceToClassesMap.contains(file)
 
     fun sourcesByInternalName(internalName: String): Collection<File> =
-        internalNameToSource[internalName]
+        internalNameToSource.getFiles(internalName)
 
     fun getAllPartsOfMultifileFacade(facade: JvmClassName): Collection<String>? {
         return multifileFacadeToParts[facade]
@@ -293,7 +293,7 @@ open class IncrementalJvmCache(
         }
 
         removeAllFromClassStorage(dirtyClasses.map { it.fqNameForClassNameWithoutDollars }, changesCollector)
-        dirtyOutputClassesMap.clean()
+        dirtyOutputClassesMap.clear()
     }
 
     override fun getObsoletePackageParts(): Collection<String> {
@@ -542,11 +542,6 @@ open class IncrementalJvmCache(
     ) :
         BasicStringMap<String>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) {
 
-        @Synchronized
-        fun set(partName: String, facadeName: String) {
-            storage[partName] = facadeName
-        }
-
         fun get(partName: JvmClassName): String? =
             storage[partName.internalName]
 
@@ -566,13 +561,9 @@ open class IncrementalJvmCache(
             storage[internalName] = pathConverter.toPaths(sourceFiles)
         }
 
-        operator fun get(internalName: String): Collection<File> =
+        fun getFiles(internalName: String): Collection<File> =
             pathConverter.toFiles(storage[internalName] ?: emptyList())
 
-        fun remove(internalName: String) {
-            storage.remove(internalName)
-        }
-
         override fun dumpValue(value: Collection<String>): String =
             value.dumpCollection()
     }
@@ -638,7 +629,7 @@ open class IncrementalJvmCache(
 }
 
 private object PathCollectionExternalizer :
-    CollectionExternalizer<String>(PathStringDescriptor, { THashSet(CollectionFactory.createFilePathSet()) })
+    CollectionExternalizerV2<String, Collection<String>>(PathStringDescriptor, { THashSet(CollectionFactory.createFilePathSet()) })
 
 sealed class ChangeInfo(val fqName: FqName) {
     open class MembersChanged(fqName: FqName, val names: Collection<String>) : ChangeInfo(fqName) {

build-common/src/org/jetbrains/kotlin/incremental/LookupStorage.kt

@@ -96,7 +96,7 @@ open class LookupStorage(
         val filtered = mutableSetOf<Int>()
 
         for (fileId in fileIds) {
-            val path = idToFile[fileId]?.path
+            val path = idToFile.getFile(fileId)?.path
 
             if (path != null) {
                 paths.add(path)
@@ -135,16 +135,16 @@ open class LookupStorage(
     }
 
     @Synchronized
-    override fun clean() {
+    override fun deleteStorageFiles() {
         icContext.transaction.deleteFile(countersFile.toPath())
 
         size = 0
 
-        super.clean()
+        super.deleteStorageFiles()
     }
 
     @Synchronized
-    override fun flush(memoryCachesOnly: Boolean) {
+    override fun close() {
         try {
             if (size != oldSize) {
                 if (size > 0) {
@@ -152,7 +152,7 @@ open class LookupStorage(
                 }
             }
         } finally {
-            super.flush(memoryCachesOnly)
+            super.close()
         }
     }
 
@@ -173,8 +173,8 @@ open class LookupStorage(
 
         val oldFileToId = fileToId.toMap()
         val oldIdToNewId = HashMap<Int, Int>(oldFileToId.size)
-        idToFile.clean()
+        idToFile.clear()
-        fileToId.clean()
+        fileToId.clear()
         size = 0
 
         for ((file, oldId) in oldFileToId.entries.sortedBy { it.key.path }) {

build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMap.kt

@@ -18,111 +18,127 @@ package org.jetbrains.kotlin.incremental.storage
 
 import com.intellij.util.io.DataExternalizer
 import com.intellij.util.io.EnumeratorStringDescriptor
+import com.intellij.util.io.IOUtil
 import com.intellij.util.io.KeyDescriptor
 import org.jetbrains.annotations.TestOnly
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.jetbrains.kotlin.utils.Printer
 import java.io.File
 
-abstract class BasicMap<K : Comparable<K>, V, StorageType : LazyStorage<K, V>>(
+/** [PersistentStorage] that provides a few extra utility methods. */
-    internal val storageFile: File,
+interface BasicMap<KEY, VALUE> : PersistentStorage<KEY, VALUE> {
-    protected val storage: StorageType,
-    protected val icContext: IncrementalCompilationContext,
-) {
-    protected val pathConverter
-        get() = icContext.pathConverter
 
-    fun clean() {
+    /** Removes all entries. */
-        storage.clean()
+    fun clear() {
+        keys.forEach { remove(it) }
     }
 
+    /**
+     * DEPRECATED: This method should be removed because
+     *   - The `memoryCachesOnly` parameter is not used (it is always `false`).
+     *   - There is currently no good use case for flushing. In fact, the current implementation of this method does nothing.
+     */
     fun flush(memoryCachesOnly: Boolean) {
-        storage.flush(memoryCachesOnly)
+        check(!memoryCachesOnly) { "Expected memoryCachesOnly = false but it is `true`" }
     }
 
-    // avoid unsynchronized close
+    /**
-    fun close() {
+     * Deletes [storageFile] or a group of files associated with [storageFile] (e.g., an implementation of [PersistentStorage] may use a
-        storage.close()
+     * [com.intellij.util.io.PersistentHashMap], which creates files such as "storageFile.tab", "storageFile.tab.len", etc.).
+     *
+     * Make sure the storage has been closed first before calling this method.
+     */
+    fun deleteStorageFiles() {
+        check(IOUtil.deleteAllFilesStartingWith(storageFile)) {
+            "Unable to delete storage file(s) with name prefix: ${storageFile.path}"
+        }
     }
 
-    @TestOnly
+    /**
-    fun closeForTest() {
+     * DEPRECATED: This method should be removed because:
+     *   - The method name is ambiguous (it may be confused with [clear], and it does not exactly describe the current implementation).
+     *   - The method currently calls close(). However, close() is often already called separately and automatically, so calling this method
+     *     means that close() will likely be called twice.
+     * Instead, just inline this method or call either close() or deleteStorageFiles() directly.
+     */
+    fun clean() {
         close()
+        deleteStorageFiles()
     }
 
     @TestOnly
     fun dump(): String {
-        return with(StringBuilder()) {
+        return StringBuilder().apply {
-            with(Printer(this)) {
+            Printer(this).apply {
                 println("${storageFile.name.substringBefore(".tab")} (${this@BasicMap::class.java.simpleName})")
                 pushIndent()
 
-                for (key in storage.keys.sorted()) {
+                for (key in keys.sortedBy { dumpKey(it) }) {
-                    println("${dumpKey(key)} -> ${dumpValue(storage[key]!!)}")
+                    println("${dumpKey(key)} -> ${dumpValue(this@BasicMap[key]!!)}")
                 }
 
                 popIndent()
             }
-
-            this
         }.toString()
     }
 
     @TestOnly
-    protected abstract fun dumpKey(key: K): String
+    fun dumpKey(key: KEY): String = key.toString()
 
     @TestOnly
-    protected abstract fun dumpValue(value: V): String
+    fun dumpValue(value: VALUE): String = value.toString()
 }
 
-abstract class NonAppendableBasicMap<K : Comparable<K>, V>(
+abstract class BasicMapBase<KEY, VALUE>(
-    storageFile: File,
+    protected val storage: PersistentStorage<KEY, VALUE>,
-    keyDescriptor: KeyDescriptor<K>,
+) : PersistentStorageWrapper<KEY, VALUE>(storage), BasicMap<KEY, VALUE>
-    valueExternalizer: DataExternalizer<V>,
-    icContext: IncrementalCompilationContext,
-) : BasicMap<K, V, LazyStorage<K, V>>(
-    storageFile,
-    createLazyStorage(storageFile, keyDescriptor, valueExternalizer, icContext),
-    icContext
-)
 
-abstract class AppendableBasicMap<K : Comparable<K>, V>(
+abstract class AppendableBasicMapBase<KEY, E, VALUE : Collection<E>>(
-    storageFile: File,
+    protected val storage: AppendablePersistentStorage<KEY, E, VALUE>,
-    keyDescriptor: KeyDescriptor<K>,
+) : AppendablePersistentStorageWrapper<KEY, E, VALUE>(storage), BasicMap<KEY, VALUE>
-    valueExternalizer: AppendableDataExternalizer<V>,
-    icContext: IncrementalCompilationContext,
-) : BasicMap<K, V, AppendableLazyStorage<K, V>>(
-    storageFile,
-    createLazyStorage(storageFile, keyDescriptor, valueExternalizer, icContext),
-    icContext
-)
 
-abstract class BasicStringMap<V>(
+abstract class AbstractBasicMap<KEY, VALUE>(
     storageFile: File,
-    keyDescriptor: KeyDescriptor<String>,
+    keyDescriptor: KeyDescriptor<KEY>,
-    valueExternalizer: DataExternalizer<V>,
+    valueExternalizer: DataExternalizer<VALUE>,
-    icContext: IncrementalCompilationContext,
+    protected val icContext: IncrementalCompilationContext,
-) : NonAppendableBasicMap<String, V>(storageFile, keyDescriptor, valueExternalizer, icContext) {
+) : BasicMapBase<KEY, VALUE>(
-    constructor(
+    createPersistentStorage(storageFile, keyDescriptor, valueExternalizer, icContext)
-        storageFile: File,
+) {
-        valueExternalizer: DataExternalizer<V>,
+    protected val pathConverter
-        icContext: IncrementalCompilationContext,
+        get() = icContext.pathConverter
-    ) : this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext)
-
-    override fun dumpKey(key: String): String = key
 }
 
-abstract class AppendableBasicStringMap<V>(
+abstract class AppendableAbstractBasicMap<KEY, E, VALUE : Collection<E>>(
+    storageFile: File,
+    keyDescriptor: KeyDescriptor<KEY>,
+    elementExternalizer: DataExternalizer<E>,
+    protected val icContext: IncrementalCompilationContext,
+) : AppendableBasicMapBase<KEY, E, VALUE>(
+    createAppendablePersistentStorage(storageFile, keyDescriptor, elementExternalizer, icContext)
+) {
+    protected val pathConverter
+        get() = icContext.pathConverter
+}
+
+abstract class BasicStringMap<VALUE>(
     storageFile: File,
     keyDescriptor: KeyDescriptor<String>,
-    valueExternalizer: AppendableDataExternalizer<V>,
+    valueExternalizer: DataExternalizer<VALUE>,
     icContext: IncrementalCompilationContext,
-) : AppendableBasicMap<String, V>(storageFile, keyDescriptor, valueExternalizer, icContext) {
+) : AbstractBasicMap<String, VALUE>(storageFile, keyDescriptor, valueExternalizer, icContext) {
-    constructor(
-        storageFile: File,
-        valueExternalizer: AppendableDataExternalizer<V>,
-        icContext: IncrementalCompilationContext,
-    ) : this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext)
 
-    override fun dumpKey(key: String): String = key
+    constructor(storageFile: File, valueExternalizer: DataExternalizer<VALUE>, icContext: IncrementalCompilationContext) :
-}
+            this(storageFile, EnumeratorStringDescriptor.INSTANCE, valueExternalizer, icContext)
+}
+
+abstract class AppendableBasicStringMap<E, VALUE : Collection<E>>(
+    storageFile: File,
+    keyDescriptor: KeyDescriptor<String>,
+    elementExternalizer: DataExternalizer<E>,
+    icContext: IncrementalCompilationContext,
+) : AppendableAbstractBasicMap<String, E, VALUE>(storageFile, keyDescriptor, elementExternalizer, icContext) {
+
+    constructor(storageFile: File, elementExternalizer: DataExternalizer<E>, icContext: IncrementalCompilationContext) :
+            this(storageFile, EnumeratorStringDescriptor.INSTANCE, elementExternalizer, icContext)
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/BasicMapsOwner.kt

@@ -21,7 +21,7 @@ import java.io.File
 import java.io.IOException
 
 open class BasicMapsOwner(val cachesDir: File) {
-    private val maps = arrayListOf<BasicMap<*, *, *>>()
+    private val maps = arrayListOf<BasicMap<*, *>>()
 
     companion object {
         val CACHE_EXTENSION = "tab"
@@ -31,25 +31,29 @@ open class BasicMapsOwner(val cachesDir: File) {
         get() = File(cachesDir, this + "." + CACHE_EXTENSION)
 
     @Synchronized
-    protected fun <K, V, S, M : BasicMap<K, V, S>> registerMap(map: M): M {
+    protected fun <K, V, M : BasicMap<K, V>> registerMap(map: M): M {
         maps.add(map)
         return map
     }
 
-    open fun clean() {
+    fun flush(memoryCachesOnly: Boolean) {
-        forEachMapSafe("clean", BasicMap<*, *, *>::clean)
-    }
-
-    open fun close() {
-        forEachMapSafe("close", BasicMap<*, *, *>::close)
-    }
-
-    open fun flush(memoryCachesOnly: Boolean) {
         forEachMapSafe("flush") { it.flush(memoryCachesOnly) }
     }
 
+    open fun close() {
+        forEachMapSafe("close", BasicMap<*, *>::close)
+    }
+
+    open fun deleteStorageFiles() {
+        forEachMapSafe("deleteStorageFiles", BasicMap<*, *>::deleteStorageFiles)
+    }
+
+    fun clean() {
+        forEachMapSafe("clean", BasicMap<*, *>::clean)
+    }
+
     @Synchronized
-    private fun forEachMapSafe(actionName: String, action: (BasicMap<*, *, *>) -> Unit) {
+    private fun forEachMapSafe(actionName: String, action: (BasicMap<*, *>) -> Unit) {
         val actionExceptions = LinkedHashMap<String, Exception>()
         maps.forEach {
             try {

build-common/src/org/jetbrains/kotlin/incremental/storage/CachingLazyStorage.kt

@@ -1,158 +0,0 @@
-/*
- * Copyright 2010-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.jetbrains.kotlin.incremental.storage
-
-import com.intellij.util.CommonProcessors
-import com.intellij.util.io.AppendablePersistentMap
-import com.intellij.util.io.DataExternalizer
-import com.intellij.util.io.IOUtil
-import com.intellij.util.io.KeyDescriptor
-import com.intellij.util.io.PersistentHashMap
-import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
-import java.io.File
-import java.io.IOException
-
-
-/**
- * It's lazy in a sense that PersistentHashMap is created only on write
- */
-class CachingLazyStorage<K, V>(
-    private val storageFile: File,
-    private val keyDescriptor: KeyDescriptor<K>,
-    private val valueExternalizer: DataExternalizer<V>
-) : AppendableLazyStorage<K, V> {
-    private var storage: PersistentHashMap<K, V>? = null
-    private var isStorageFileExist = true
-
-    private fun getStorageIfExists(): PersistentHashMap<K, V>? {
-        if (storage != null) return storage
-
-        if (!isStorageFileExist) return null
-
-        if (storageFile.exists()) {
-            storage = createMap()
-            return storage
-        }
-
-        isStorageFileExist = false
-        return null
-    }
-
-    private fun getStorageOrCreateNew(): PersistentHashMap<K, V> {
-        if (storage == null) {
-            storage = createMap()
-        }
-        return storage!!
-    }
-
-    override val keys: Collection<K>
-        @Synchronized
-        get() = buildList {
-            getStorageIfExists()?.processKeysWithExistingMapping(CommonProcessors.CollectProcessor(this))
-        }
-
-    @Synchronized
-    override operator fun contains(key: K): Boolean =
-        getStorageIfExists()?.containsMapping(key) ?: false
-
-    @Synchronized
-    override operator fun get(key: K): V? =
-        getStorageIfExists()?.get(key)
-
-    @Synchronized
-    override operator fun set(key: K, value: V) {
-        getStorageOrCreateNew().put(key, value)
-    }
-
-    @Synchronized
-    override fun remove(key: K) {
-        getStorageIfExists()?.remove(key)
-    }
-
-    @Synchronized
-    override fun append(key: K, value: V) {
-        check(valueExternalizer is AppendableDataExternalizer<*>) {
-            "`valueExternalizer` should implement the `AppendableDataExternalizer` interface to be able to call `append`"
-        }
-        getStorageOrCreateNew().appendData(key, AppendablePersistentMap.ValueDataAppender { valueExternalizer.save(it, value) })
-    }
-
-    @Synchronized
-    override fun clean() {
-        try {
-            storage?.close()
-        } finally {
-            storage = null
-            if (!IOUtil.deleteAllFilesStartingWith(storageFile)) {
-                throw IOException("Could not delete internal storage: ${storageFile.absolutePath}")
-            }
-        }
-    }
-
-    @Synchronized
-    override fun flush(memoryCachesOnly: Boolean) {
-        val existingStorage = storage ?: return
-
-        if (memoryCachesOnly) {
-            if (existingStorage.isDirty) {
-                existingStorage.dropMemoryCaches()
-            }
-        } else {
-            existingStorage.force()
-        }
-    }
-
-    @Synchronized
-    override fun close() {
-        try {
-            storage?.close()
-        } finally {
-            storage = null
-        }
-    }
-
-    private fun createMap(): PersistentHashMap<K, V> = PersistentHashMap(storageFile, keyDescriptor, valueExternalizer)
-}
-
-private fun <K, V> createLazyStorageImpl(
-    storageFile: File,
-    keyDescriptor: KeyDescriptor<K>,
-    valueExternalizer: DataExternalizer<V>,
-    icContext: IncrementalCompilationContext,
-) = CachingLazyStorage(storageFile, keyDescriptor, valueExternalizer).let {
-    if (icContext.keepIncrementalCompilationCachesInMemory) {
-        DefaultInMemoryStorageWrapper(it, valueExternalizer).also { wrapper ->
-            icContext.transaction.registerInMemoryStorageWrapper(wrapper)
-        }
-    } else {
-        it
-    }
-}
-
-fun <K, V> createLazyStorage(
-    storageFile: File,
-    keyDescriptor: KeyDescriptor<K>,
-    valueExternalizer: DataExternalizer<V>,
-    icContext: IncrementalCompilationContext,
-): LazyStorage<K, V> = createLazyStorageImpl(storageFile, keyDescriptor, valueExternalizer, icContext)
-
-fun <K, V> createLazyStorage(
-    storageFile: File,
-    keyDescriptor: KeyDescriptor<K>,
-    valueExternalizer: AppendableDataExternalizer<V>,
-    icContext: IncrementalCompilationContext,
-): AppendableLazyStorage<K, V> = createLazyStorageImpl(storageFile, keyDescriptor, valueExternalizer, icContext)

build-common/src/org/jetbrains/kotlin/incremental/storage/ClassOneToManyMap.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
+import com.intellij.util.io.EnumeratorStringDescriptor
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.jetbrains.kotlin.incremental.dumpCollection
 import org.jetbrains.kotlin.name.FqName
@@ -24,17 +25,17 @@ import java.io.File
 internal open class ClassOneToManyMap(
     storageFile: File,
     icContext: IncrementalCompilationContext,
-) : AppendableBasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer, icContext) {
+) : AppendableBasicStringMap<String, Collection<String>>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) {
-    override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
+    override fun dumpValue(value: Collection<String>): String = value.toSet().dumpCollection()
 
     @Synchronized
     fun add(key: FqName, value: FqName) {
-        storage.append(key.asString(), listOf(value.asString()))
+        storage.append(key.asString(), value.asString())
     }
 
     @Synchronized
     operator fun get(key: FqName): Collection<FqName> =
-        storage[key.asString()]?.map(::FqName) ?: setOf()
+        storage[key.asString()]?.mapTo(mutableSetOf(), ::FqName) ?: setOf()
 
     @Synchronized
     operator fun set(key: FqName, values: Collection<FqName>) {

build-common/src/org/jetbrains/kotlin/incremental/storage/ComplementarySourceFilesMap.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
+import com.intellij.util.io.EnumeratorStringDescriptor
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.jetbrains.kotlin.incremental.dumpCollection
 import java.io.File
@@ -12,7 +13,12 @@ import java.io.File
 class ComplementarySourceFilesMap(
     storageFile: File,
     icContext: IncrementalCompilationContext,
-) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) {
+) : AppendableBasicStringMap<String, Collection<String>>(
+    storageFile,
+    PathStringDescriptor,
+    EnumeratorStringDescriptor.INSTANCE,
+    icContext
+) {
 
     operator fun set(sourceFile: File, complementaryFiles: Collection<File>) {
         storage[pathConverter.toPath(sourceFile)] = pathConverter.toPaths(complementaryFiles)
@@ -20,7 +26,7 @@ class ComplementarySourceFilesMap(
 
     operator fun get(sourceFile: File): Collection<File> {
         val paths = storage[pathConverter.toPath(sourceFile)].orEmpty()
-        return pathConverter.toFiles(paths)
+        return pathConverter.toFiles(paths).toSet()
     }
 
     override fun dumpValue(value: Collection<String>) =

build-common/src/org/jetbrains/kotlin/incremental/storage/IdToFileMap.kt

@@ -24,20 +24,15 @@ import java.io.File
 internal class IdToFileMap(
     file: File,
     icContext: IncrementalCompilationContext,
-) : NonAppendableBasicMap<Int, String>(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE, icContext) {
+) : AbstractBasicMap<Int, String>(file, ExternalIntegerKeyDescriptor(), EnumeratorStringDescriptor.INSTANCE, icContext) {
     override fun dumpKey(key: Int): String = key.toString()
 
     override fun dumpValue(value: String): String = value
 
-    operator fun get(id: Int): File? = storage[id]?.let { pathConverter.toFile(it) }
+    fun getFile(id: Int): File? = storage[id]?.let { pathConverter.toFile(it) }
-
-    operator fun contains(id: Int): Boolean = id in storage
 
     operator fun set(id: Int, file: File) {
         storage[id] = pathConverter.toPath(file)
     }
 
-    fun remove(id: Int) {
-        storage.remove(id)
-    }
 }

build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorage.kt

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.storage
+
+import org.jetbrains.kotlin.utils.ThreadSafe
+
+/**
+ * [PersistentStorage] which reads from another underlying [PersistentStorage], keeps all changes to it in memory, and writes the changes
+ * back to the underlying storage on [applyChanges] or [close].
+ */
+interface InMemoryStorageInterface<KEY, VALUE> : PersistentStorage<KEY, VALUE> {
+
+    /** Applies in-memory changes to the underlying [PersistentStorage], then calls [clearChanges]. */
+    fun applyChanges()
+
+    /** Removes all in-memory changes. */
+    fun clearChanges()
+}
+
+/** See [InMemoryStorageInterface]. */
+@ThreadSafe
+open class InMemoryStorage<KEY, VALUE>(
+    private val storage: PersistentStorage<KEY, VALUE>,
+) : InMemoryStorageInterface<KEY, VALUE> {
+
+    override val storageFile = storage.storageFile
+
+    // The following properties store changes to `storage`.
+    // Note that:
+    //   - The keys across these groups must be mutually exclusive.
+    //   - `addedEntries.keys` must be outside `storage.keys`.
+    //   - `modifiedEntries.keys`, `appendedEntries.keys`, and `removedKeys` must be subsets of `storage.keys`.
+    //   - `appendedEntries` is meant to be used only by the subclass `AppendableInMemoryStorage`, but it is in this class because we want
+    //     to share code as much as possible.
+    protected val addedEntries = LinkedHashMap<KEY, VALUE>()
+    protected val modifiedEntries = LinkedHashMap<KEY, VALUE>()
+    protected val appendedEntries = LinkedHashMap<KEY, VALUE>()
+    protected val removedKeys = LinkedHashSet<KEY>()
+
+    @get:Synchronized
+    override val keys: Set<KEY>
+        get() = storage.keys + addedEntries.keys - removedKeys
+
+    @Synchronized
+    override fun contains(key: KEY): Boolean = when (key) {
+        in addedEntries -> true
+        in modifiedEntries -> true
+        in appendedEntries -> true
+        in removedKeys -> false
+        else -> key in storage
+    }
+
+    @Synchronized
+    override fun get(key: KEY): VALUE? = addedEntries[key] ?: modifiedEntries[key] ?: when (key) {
+        in appendedEntries -> {
+            @Suppress("UNCHECKED_CAST")
+            ((storage[key]!! as Collection<*>) + (appendedEntries[key]!! as Collection<*>)) as VALUE
+        }
+        in removedKeys -> null
+        else -> storage[key]
+    }
+
+    @Synchronized
+    override fun set(key: KEY, value: VALUE) = when (key) {
+        in addedEntries -> addedEntries[key] = value
+        in modifiedEntries -> modifiedEntries[key] = value
+        in appendedEntries -> {
+            appendedEntries.remove(key)
+            modifiedEntries[key] = value
+        }
+        in removedKeys -> {
+            removedKeys.remove(key)
+            modifiedEntries[key] = value
+        }
+        in storage -> modifiedEntries[key] = value
+        else -> addedEntries[key] = value
+    }
+
+    @Synchronized
+    override fun remove(key: KEY) {
+        when (key) {
+            in addedEntries -> addedEntries.remove(key)
+            in modifiedEntries -> {
+                modifiedEntries.remove(key)
+                removedKeys.add(key)
+            }
+            in appendedEntries -> {
+                appendedEntries.remove(key)
+                removedKeys.add(key)
+            }
+            in removedKeys -> Unit
+            in storage -> removedKeys.add(key)
+            else -> Unit
+        }
+    }
+
+    @Synchronized
+    override fun applyChanges() {
+        addedEntries.forEach {
+            storage[it.key] = it.value
+        }
+        modifiedEntries.forEach {
+            storage[it.key] = it.value
+        }
+        check(appendedEntries.isEmpty()) { "appendedEntries is not empty" }
+        removedKeys.forEach {
+            storage.remove(it)
+        }
+        clearChanges()
+    }
+
+    @Synchronized
+    override fun clearChanges() {
+        addedEntries.clear()
+        modifiedEntries.clear()
+        appendedEntries.clear()
+        removedKeys.clear()
+    }
+
+    @Synchronized
+    override fun close() {
+        applyChanges()
+        storage.close()
+    }
+
+}
+
+/** [InMemoryStorage] where a map entry's value is a [Collection]. */
+@ThreadSafe
+class AppendableInMemoryStorage<KEY, E, VALUE : Collection<E>>(
+    private val storage: AppendablePersistentStorage<KEY, E, VALUE>,
+) : InMemoryStorage<KEY, VALUE>(storage), AppendablePersistentStorage<KEY, E, VALUE> {
+
+    @Suppress("UNCHECKED_CAST")
+    @Synchronized
+    override fun append(key: KEY, elements: VALUE) = when (key) {
+        in addedEntries -> addedEntries[key] = (addedEntries[key]!! + elements) as VALUE
+        in modifiedEntries -> modifiedEntries[key] = (modifiedEntries[key]!! + elements) as VALUE
+        in appendedEntries -> appendedEntries[key] = (appendedEntries[key]!! + elements) as VALUE
+        in removedKeys -> {
+            removedKeys.remove(key)
+            // Note: We update `modifiedEntries` (not `appendedEntries`), because if the entry is removed and then appended, it is
+            // equivalent to being modified (not appended)
+            modifiedEntries[key] = elements
+        }
+        in storage -> appendedEntries[key] = elements
+        else -> addedEntries[key] = elements
+    }
+
+    @Synchronized
+    override fun applyChanges() {
+        appendedEntries.forEach {
+            storage.append(it.key, it.value)
+        }
+        appendedEntries.clear()
+        super.applyChanges()
+    }
+
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/InMemoryStorageWrapper.kt

@@ -1,171 +0,0 @@
-/*
- * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.incremental.storage
-
-import com.intellij.util.io.DataExternalizer
-import java.util.*
-import kotlin.collections.LinkedHashMap
-
-interface InMemoryStorageWrapper<K, V> : AppendableLazyStorage<K, V> {
-    fun resetInMemoryChanges()
-}
-
-/**
- * An in-memory wrapper for [origin] that keeps all the write operations in-memory.
- * Flushes all the changes to the [origin] on [flush] invocation.
- * [resetInMemoryChanges] should be called to reset in-memory changes of this wrapper.
- */
-class DefaultInMemoryStorageWrapper<K, V>(
-    private val origin: CachingLazyStorage<K, V>,
-    private val valueExternalizer: DataExternalizer<V>
-) :
-    InMemoryStorageWrapper<K, V> {
-    // These state properties keep the current diff that will be applied to the [origin] on flush if [resetInMemoryChanges] is not called
-    private val inMemoryStorage = LinkedHashMap<K, ValueWrapper>()
-    private val removedKeys = hashSetOf<K>()
-    private var isCleanRequested = false
-
-    @get:Synchronized
-    override val keys: Collection<K>
-        get() = if (isCleanRequested) inMemoryStorage.keys else (origin.keys - removedKeys) + inMemoryStorage.keys
-
-    @Synchronized
-    override fun resetInMemoryChanges() {
-        isCleanRequested = false
-        inMemoryStorage.clear()
-        removedKeys.clear()
-    }
-
-    @Synchronized
-    override fun clean() {
-        inMemoryStorage.clear()
-        removedKeys.clear()
-        isCleanRequested = true
-    }
-
-    @Synchronized
-    override fun flush(memoryCachesOnly: Boolean) {
-        if (isCleanRequested) {
-            origin.clean()
-        } else {
-            for (key in removedKeys) {
-                origin.remove(key)
-            }
-        }
-        for ((key, valueWrapper) in inMemoryStorage) {
-            when (valueWrapper) {
-                is ValueWrapper.Value<*> -> origin[key] = valueWrapper.value.cast()
-                // if we were appending the value and didn't access it,
-                // then we have it as an append chain, so merge it and append to the origin as a single value
-                is ValueWrapper.AppendChain<*> -> origin.append(key, getMergedValue(key, valueWrapper, false)).also {
-                    origin[key] // trigger chunks compaction
-                }
-            }
-        }
-
-        resetInMemoryChanges()
-
-        origin.flush(memoryCachesOnly)
-    }
-
-    @Synchronized
-    override fun close() {
-        origin.close()
-    }
-
-    @Synchronized
-    override fun append(key: K, value: V) {
-        /*
-         * Plain English explanation:
-         * 1. The key's value is present only in origin => appendToOrigin = true
-         * 2. The key's value was set in this wrapper => appendToOrigin = false
-         * 3. The key's value was appended but not set in this wrapper => appendToOrigin = true
-         */
-        check(valueExternalizer is AppendableDataExternalizer<V>) {
-            "`valueExternalizer` should implement the `AppendableDataExternalizer` interface to be able to call `append`"
-        }
-        val currentWrapper = inMemoryStorage[key]
-        if (currentWrapper is ValueWrapper.AppendChain<*>) {
-            (currentWrapper.parts.cast<MutableList<V>>()).add(value)
-            return
-        }
-
-        val newWrapper = when (currentWrapper) {
-            is ValueWrapper.Value<*> -> ValueWrapper.AppendChain(mutableListOf(currentWrapper.value.cast(), value), false)
-            // if `append` is called for the first time, assume it will be called more, so don't store it as `ValueWrapper.Value`
-            else -> ValueWrapper.AppendChain(mutableListOf(value), true)
-        }
-
-        inMemoryStorage[key] = newWrapper
-    }
-
-    @Synchronized
-    override fun remove(key: K) {
-        removedKeys.add(key)
-        inMemoryStorage.remove(key)
-    }
-
-    @Synchronized
-    override fun set(key: K, value: V) {
-        inMemoryStorage[key] = ValueWrapper.Value(value)
-    }
-
-    @Synchronized
-    override fun get(key: K): V? {
-        val wrapper = inMemoryStorage[key]
-        return when {
-            wrapper is ValueWrapper.Value<*> -> wrapper.value.cast<V>()
-            wrapper is ValueWrapper.AppendChain<*> -> getMergedValue(key, wrapper).also { mergedValue ->
-                inMemoryStorage[key] = ValueWrapper.Value(mergedValue)
-            }
-            key !in removedKeys -> origin[key]
-            else -> null
-        }
-    }
-
-    @Synchronized
-    override fun contains(key: K) = key in inMemoryStorage || (key !in removedKeys && key in origin)
-
-    /**
-     * Merges a value for a [key] from [origin] if it isn't in [removedKeys] and [useOriginValue] != false with [ValueWrapper.AppendChain] and returns the merged value
-     */
-    private fun getMergedValue(key: K, wrapper: ValueWrapper, useOriginValue: Boolean = true): V {
-        check(wrapper !is ValueWrapper.Value<*>) {
-            "There's no need to merge values for $key"
-        }
-        check(valueExternalizer is AppendableDataExternalizer<V>) {
-            "`valueExternalizer` should implement the `AppendableDataExternalizer` interface to be able to handle `append`"
-        }
-        return when (wrapper) {
-            is ValueWrapper.AppendChain<*> -> {
-                fun merge(acc: V, append: V) = valueExternalizer.append(acc, append)
-
-                val initial = if (useOriginValue && wrapper.appendToOrigin) {
-                    listOfNotNull(getOriginValue(key)).fold(valueExternalizer.createNil(), ::merge)
-                } else {
-                    valueExternalizer.createNil()
-                }
-                (wrapper.parts.cast<MutableList<V>>()).fold(initial, ::merge)
-            }
-            else -> error("In-memory storage contains no value for $key")
-        }
-    }
-
-    private fun getOriginValue(key: K): V? = if (key !in removedKeys) {
-        origin[key]
-    } else {
-        null
-    }
-
-    @Suppress("UNCHECKED_CAST")
-    private fun <T> Any?.cast() = this as T
-
-    private sealed interface ValueWrapper {
-        class Value<V>(val value: V) : ValueWrapper
-
-        class AppendChain<V>(val parts: MutableList<V>, val appendToOrigin: Boolean) : ValueWrapper
-    }
-}

build-common/src/org/jetbrains/kotlin/incremental/storage/LazyStorage.kt

@@ -16,19 +16,162 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
-import java.io.Closeable
+import com.intellij.util.CommonProcessors
+import com.intellij.util.io.AppendablePersistentMap
+import com.intellij.util.io.DataExternalizer
+import com.intellij.util.io.KeyDescriptor
+import com.intellij.util.io.PersistentHashMap
+import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
+import org.jetbrains.kotlin.utils.ThreadSafe
+import java.io.DataInput
+import java.io.DataInputStream
+import java.io.DataOutput
+import java.io.File
+
+/**
+ * [PersistentStorage] which delegates operations to a [PersistentHashMap]. Note that the [PersistentHashMap] is created lazily (only when
+ * required).
+ */
+@ThreadSafe
+open class LazyStorage<KEY, VALUE>(
+    override val storageFile: File,
+    private val keyDescriptor: KeyDescriptor<KEY>,
+    private val valueExternalizer: DataExternalizer<VALUE>,
+) : PersistentStorage<KEY, VALUE> {
+
+    private var storage: PersistentHashMap<KEY, VALUE>? = null
+
+    // Use this property to minimize I/O
+    private val isStorageFileExist: Boolean by lazy {
+        storageFile.exists()
+    }
+
+    private fun getStorageIfExists(): PersistentHashMap<KEY, VALUE>? {
+        return storage ?: when {
+            isStorageFileExist -> createMap().also { storage = it }
+            else -> null
+        }
+    }
+
+    protected fun getStorageOrCreateNew(): PersistentHashMap<KEY, VALUE> {
+        return storage ?: createMap().also { storage = it }
+    }
+
+    private fun createMap() = PersistentHashMap(storageFile, keyDescriptor, valueExternalizer)
+
+    @get:Synchronized
+    override val keys: Set<KEY>
+        get() = buildSet {
+            getStorageIfExists()?.processKeysWithExistingMapping(CommonProcessors.CollectProcessor(this))
+        }
+
+    @Synchronized
+    override fun contains(key: KEY): Boolean =
+        getStorageIfExists()?.containsMapping(key) ?: false
+
+    @Synchronized
+    override fun get(key: KEY): VALUE? =
+        getStorageIfExists()?.get(key)
+
+    @Synchronized
+    override fun set(key: KEY, value: VALUE) {
+        getStorageOrCreateNew().put(key, value)
+    }
+
+    @Synchronized
+    override fun remove(key: KEY) {
+        getStorageIfExists()?.remove(key)
+    }
+
+    @Synchronized
+    override fun close() {
+        storage?.close()
+    }
 
-interface LazyStorage<K, V> : Closeable {
-    val keys: Collection<K>
-    operator fun contains(key: K): Boolean
-    operator fun get(key: K): V?
-    operator fun set(key: K, value: V)
-    fun remove(key: K)
-    fun clean()
-    fun flush(memoryCachesOnly: Boolean)
-    override fun close()
 }
 
-interface AppendableLazyStorage<K, V> : LazyStorage<K, V> {
+/** [LazyStorage] where a map entry's value is a [Collection]. */
-    fun append(key: K, value: V)
+@ThreadSafe
-}
+class AppendableLazyStorage<KEY, E, VALUE : Collection<E>>(
+    storageFile: File,
+    keyDescriptor: KeyDescriptor<KEY>,
+    elementExternalizer: DataExternalizer<E>,
+) : LazyStorage<KEY, VALUE>(storageFile, keyDescriptor, AppendableCollectionExternalizer(elementExternalizer)),
+    AppendablePersistentStorage<KEY, E, VALUE> {
+
+    private val appendableCollectionExternalizer = AppendableCollectionExternalizer(elementExternalizer)
+
+    @Synchronized
+    override fun append(key: KEY, elements: VALUE) {
+        getStorageOrCreateNew().appendData(
+            key,
+            AppendablePersistentMap.ValueDataAppender { appendableCollectionExternalizer.append(it, elements) }
+        )
+    }
+}
+
+/**
+ * [DataExternalizer] for a [Collection] of type [E].
+ *
+ * IMPORTANT: It is a *private* class because it is meant to be used only with a [PersistentHashMap] (e.g., the [read] method reads until
+ * the stream ends, [append] can be called multiple times and its implementation is identical to [save] -- these only work with a
+ * [PersistentHashMap]).
+ */
+private class AppendableCollectionExternalizer<E, VALUE : Collection<E>>(
+    private val elementExternalizer: DataExternalizer<E>,
+) : DataExternalizer<VALUE> {
+
+    fun append(output: DataOutput, elements: VALUE) {
+        save(output, elements)
+    }
+
+    override fun save(output: DataOutput, value: VALUE) {
+        value.forEach { elementExternalizer.save(output, it) }
+    }
+
+    override fun read(input: DataInput): VALUE {
+        val result = ArrayList<E>()
+        val stream = input as DataInputStream
+
+        while (stream.available() > 0) {
+            result.add(elementExternalizer.read(stream))
+        }
+
+        @Suppress("UNCHECKED_CAST")
+        return result as VALUE
+    }
+}
+
+fun <KEY, VALUE> createPersistentStorage(
+    storageFile: File,
+    keyDescriptor: KeyDescriptor<KEY>,
+    valueExternalizer: DataExternalizer<VALUE>,
+    icContext: IncrementalCompilationContext,
+): PersistentStorage<KEY, VALUE> {
+    return LazyStorage(storageFile, keyDescriptor, valueExternalizer).let { storage ->
+        if (icContext.keepIncrementalCompilationCachesInMemory) {
+            InMemoryStorage(storage).also {
+                icContext.transaction.registerInMemoryStorageWrapper(it)
+            }
+        } else {
+            storage
+        }
+    }
+}
+
+fun <KEY, E, VALUE : Collection<E>> createAppendablePersistentStorage(
+    storageFile: File,
+    keyDescriptor: KeyDescriptor<KEY>,
+    elementExternalizer: DataExternalizer<E>,
+    icContext: IncrementalCompilationContext,
+): AppendablePersistentStorage<KEY, E, VALUE> {
+    return AppendableLazyStorage<KEY, E, VALUE>(storageFile, keyDescriptor, elementExternalizer).let { storage ->
+        if (icContext.keepIncrementalCompilationCachesInMemory) {
+            AppendableInMemoryStorage(storage).also {
+                icContext.transaction.registerInMemoryStorageWrapper(it)
+            }
+        } else {
+            storage
+        }
+    }
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/LookupMap.kt

@@ -23,10 +23,10 @@ class LookupMap(
     storage: File,
     icContext: IncrementalCompilationContext,
 ) :
-    AppendableBasicMap<LookupSymbolKey, Collection<Int>>(
+    AppendableAbstractBasicMap<LookupSymbolKey, Int, Collection<Int>>(
         storage,
         LookupSymbolKeyDescriptor(icContext.storeFullFqNamesInLookupCache),
-        IntCollectionExternalizer,
+        IntExternalizer,
         icContext,
     ) {
 
@@ -35,23 +35,13 @@ class LookupMap(
     override fun dumpValue(value: Collection<Int>): String = value.toString()
 
     fun add(name: String, scope: String, fileId: Int) {
-        storage.append(LookupSymbolKey(name, scope), listOf(fileId))
+        storage.append(LookupSymbolKey(name, scope), fileId)
     }
 
-    fun append(lookup: LookupSymbolKey, fileIds: Collection<Int>) {
+    override fun get(key: LookupSymbolKey): Collection<Int>? = storage[key]?.toSet()
-        storage.append(lookup, fileIds)
-    }
-
-    operator fun get(key: LookupSymbolKey): Collection<Int>? = storage[key]
 
     operator fun set(key: LookupSymbolKey, fileIds: Set<Int>) {
         storage[key] = fileIds
     }
 
-    fun remove(key: LookupSymbolKey) {
-        storage.remove(key)
-    }
-
-    val keys: Collection<LookupSymbolKey>
-        get() = storage.keys
 }

build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorage.kt

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2010-2015 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.incremental.storage
+
+import org.jetbrains.kotlin.utils.ThreadSafe
+import java.io.Closeable
+import java.io.File
+
+/**
+ * Represents an in-memory map that is backed by a [storageFile].
+ *
+ * Changes to this map may be written to [storageFile] at any time, and it is guaranteed to be written on [close].
+ *
+ * This interface is similar to but simpler than [com.intellij.util.io.PersistentMapBase].
+ */
+@Suppress("UnstableApiUsage")
+interface PersistentStorage<KEY, VALUE> : Closeable {
+
+    /** The storage file backing this map. */
+    val storageFile: File
+
+    val keys: Set<KEY>
+
+    operator fun contains(key: KEY): Boolean
+
+    operator fun get(key: KEY): VALUE?
+
+    operator fun set(key: KEY, value: VALUE)
+
+    fun remove(key: KEY)
+
+    /** Writes any remaining in-memory changes to [storageFile] and closes this map. */
+    override fun close()
+}
+
+/** [PersistentStorage] where a map entry's value is a [Collection]. */
+interface AppendablePersistentStorage<KEY, E, VALUE : Collection<E>> : PersistentStorage<KEY, VALUE> {
+
+    /** Adds the given [elements] to the collection corresponding to the given [key]. */
+    fun append(key: KEY, elements: VALUE)
+
+    /** Adds the given [element] to the collection corresponding to the given [key]. */
+    fun append(key: KEY, element: E) {
+        @Suppress("UNCHECKED_CAST")
+        append(key, listOf(element) as VALUE)
+    }
+}
+
+/**
+ * [PersistentStorage] that delegates operations to another [storage].
+ *
+ * THREAD SAFETY: All [PersistentStorage]s (both parent classes and their subclasses) need to be thread-safe. This requirement seems to come
+ * from JPS -- see commit 275a02c; Gradle builds don't have this requirement. To ensure thread safety, currently all non-private
+ * implementation methods of [PersistentStorage]s and their subclasses must be `@Synchronized`. A possibly better approach is to perform
+ * synchronization in JPS, so that we don't have to provide the thread safety guarantee for [PersistentStorage]s.
+ */
+@ThreadSafe
+abstract class PersistentStorageWrapper<KEY, VALUE>(
+    private val storage: PersistentStorage<KEY, VALUE>,
+) : PersistentStorage<KEY, VALUE> { // Can't use Kotlin delegation (`by storage`) here as we need to annotate the methods with @Synchronized
+
+    override val storageFile: File = storage.storageFile
+
+    @get:Synchronized
+    override val keys: Set<KEY>
+        get() = storage.keys
+
+    @Synchronized
+    override fun contains(key: KEY): Boolean =
+        storage.contains(key)
+
+    @Synchronized
+    override fun get(key: KEY): VALUE? =
+        storage[key]
+
+    @Synchronized
+    override fun set(key: KEY, value: VALUE) {
+        storage[key] = value
+    }
+
+    @Synchronized
+    override fun remove(key: KEY) {
+        storage.remove(key)
+    }
+
+    @Synchronized
+    override fun close() {
+        storage.close()
+    }
+}
+
+/** [PersistentStorageWrapper] where a map entry's value is a [Collection]. */
+@ThreadSafe
+abstract class AppendablePersistentStorageWrapper<KEY, E, VALUE : Collection<E>>(
+    private val appendableStorage: AppendablePersistentStorage<KEY, E, VALUE>,
+) : PersistentStorageWrapper<KEY, VALUE>(appendableStorage), AppendablePersistentStorage<KEY, E, VALUE> {
+
+    @Synchronized
+    override fun append(key: KEY, elements: VALUE) {
+        appendableStorage.append(key, elements)
+    }
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/PersistentStorageAdapter.kt

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.incremental.storage
+
+import org.jetbrains.kotlin.utils.ThreadSafe
+import java.io.File
+
+/**
+ * [PersistentStorage] which maps [KEY] to [VALUE] as viewed by the users of this class, but delegates operations to another [storage] which
+ * maps [INTERNAL_KEY] to [INTERNAL_VALUE].
+ *
+ * The users of this class need to provide the transformations from [KEY] to [INTERNAL_KEY], [VALUE] to [INTERNAL_VALUE], and vice versa.
+ */
+@ThreadSafe
+abstract class PersistentStorageAdapter<KEY, VALUE, INTERNAL_KEY, INTERNAL_VALUE>(
+    private val storage: PersistentStorage<INTERNAL_KEY, INTERNAL_VALUE>,
+    private val publicToInternalKey: (KEY) -> INTERNAL_KEY,
+    private val internalToPublicKey: (INTERNAL_KEY) -> KEY,
+    private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE,
+    private val internalToPublicValue: (INTERNAL_VALUE) -> VALUE,
+) : PersistentStorage<KEY, VALUE>, BasicMap<KEY, VALUE> {
+
+    override val storageFile: File = storage.storageFile
+
+    @get:Synchronized
+    override val keys: Set<KEY>
+        get() = storage.keys.mapTo(LinkedHashSet()) { internalToPublicKey(it) }
+
+    @Synchronized
+    override fun contains(key: KEY): Boolean =
+        storage.contains(publicToInternalKey(key))
+
+    @Synchronized
+    override fun get(key: KEY): VALUE? =
+        storage[publicToInternalKey(key)]?.let { internalToPublicValue(it) }
+
+    @Synchronized
+    override fun set(key: KEY, value: VALUE) {
+        storage[publicToInternalKey(key)] = publicToInternalValue(value)
+    }
+
+    @Synchronized
+    override fun remove(key: KEY) {
+        storage.remove(publicToInternalKey(key))
+    }
+
+    @Synchronized
+    override fun close() {
+        storage.close()
+    }
+}
+
+/** [PersistentStorageAdapter] where a map entry's value is a [Collection]. */
+@ThreadSafe
+abstract class AppendablePersistentStorageAdapter<KEY, E, VALUE : Collection<E>, INTERNAL_KEY, INTERNAL_E, INTERNAL_VALUE : Collection<INTERNAL_E>>(
+    private val storage: AppendablePersistentStorage<INTERNAL_KEY, INTERNAL_E, INTERNAL_VALUE>,
+    private val publicToInternalKey: (KEY) -> INTERNAL_KEY,
+    internalToPublicKey: (INTERNAL_KEY) -> KEY,
+    private val publicToInternalValue: (VALUE) -> INTERNAL_VALUE,
+    internalToPublicValue: (INTERNAL_VALUE) -> VALUE,
+) : PersistentStorageAdapter<KEY, VALUE, INTERNAL_KEY, INTERNAL_VALUE>(
+    storage,
+    publicToInternalKey,
+    internalToPublicKey,
+    publicToInternalValue,
+    internalToPublicValue
+), AppendablePersistentStorage<KEY, E, VALUE> {
+
+    @Synchronized
+    override fun append(key: KEY, elements: VALUE) {
+        storage.append(publicToInternalKey(key), publicToInternalValue(elements))
+    }
+}

build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToJsOutputMap.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
+import com.intellij.util.io.EnumeratorStringDescriptor
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.jetbrains.kotlin.incremental.dumpCollection
 import java.io.File
@@ -12,16 +13,16 @@ import java.io.File
 class SourceToJsOutputMap(
     storageFile: File,
     icContext: IncrementalCompilationContext,
-) : AppendableBasicStringMap<Collection<String>>(storageFile, StringCollectionExternalizer, icContext) {
+) : AppendableBasicStringMap<String, Collection<String>>(storageFile, EnumeratorStringDescriptor.INSTANCE, icContext) {
     override fun dumpValue(value: Collection<String>): String = value.dumpCollection()
 
     @Synchronized
     fun add(key: File, value: File) {
-        storage.append(pathConverter.toPath(key), listOf(pathConverter.toPath(value)))
+        storage.append(pathConverter.toPath(key), pathConverter.toPath(value))
     }
 
     operator fun get(sourceFile: File): Collection<File> =
-        storage[pathConverter.toPath(sourceFile)]?.map { pathConverter.toFile(it) } ?: setOf()
+        storage[pathConverter.toPath(sourceFile)]?.mapTo(mutableSetOf()) { pathConverter.toFile(it) } ?: setOf()
 
 
     @Synchronized
@@ -31,7 +32,7 @@ class SourceToJsOutputMap(
             return
         }
 
-        storage[pathConverter.toPath(key)] = values.map { pathConverter.toPath(it) }
+        storage[pathConverter.toPath(key)] = values.toSet().map { pathConverter.toPath(it) }
     }
 
     @Synchronized

build-common/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputMaps.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.incremental.storage
 
+import com.intellij.util.io.EnumeratorStringDescriptor
 import org.jetbrains.kotlin.incremental.IncrementalCompilationContext
 import org.jetbrains.kotlin.incremental.dumpCollection
 import org.jetbrains.kotlin.name.FqName
@@ -36,28 +37,30 @@ internal abstract class AbstractSourceToOutputMap<Name>(
     private val nameTransformer: NameTransformer<Name>,
     storageFile: File,
     icContext: IncrementalCompilationContext,
-) : AppendableBasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) {
+) : AppendableBasicStringMap<String, Collection<String>>(
+    storageFile,
+    PathStringDescriptor,
+    EnumeratorStringDescriptor.INSTANCE,
+    icContext
+) {
     fun clearOutputsForSource(sourceFile: File) {
         remove(pathConverter.toPath(sourceFile))
     }
 
     fun add(sourceFile: File, className: Name) {
-        storage.append(pathConverter.toPath(sourceFile), listOf(nameTransformer.asString(className)))
+        storage.append(pathConverter.toPath(sourceFile), nameTransformer.asString(className))
     }
 
     fun contains(sourceFile: File): Boolean =
         pathConverter.toPath(sourceFile) in storage
 
     operator fun get(sourceFile: File): Collection<Name> =
-        storage[pathConverter.toPath(sourceFile)].orEmpty().map(nameTransformer::asName)
+        storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asName)
 
     fun getFqNames(sourceFile: File): Collection<FqName> =
-        storage[pathConverter.toPath(sourceFile)].orEmpty().map(nameTransformer::asFqName)
+        storage[pathConverter.toPath(sourceFile)].orEmpty().toSet().map(nameTransformer::asFqName)
 
     override fun dumpValue(value: Collection<String>) =
         value.dumpCollection()
 
-    private fun remove(path: String) {
-        storage.remove(path)
-    }
 }

build-common/src/org/jetbrains/kotlin/incremental/storage/externalizers.kt

@@ -31,23 +31,6 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.jvm.JvmClassName
 import java.io.*
 
-/**
- * Externalizer that works correctly when [com.intellij.util.io.PersistentHashMap.appendData] is called
- *
- * Besides the [append] method, it should support incremental [save] and [read]. E.g. if [save] was called multiple times, [read] should be able to collect them together
- */
-interface AppendableDataExternalizer<T> : DataExternalizer<T> {
-    /**
-     * Creates an empty appendable object
-     */
-    fun createNil(): T
-
-    /**
-     * Combines two non-serialized appendable objects
-     */
-    fun append(currentValue: T, appendData: T): T
-}
-
 class LookupSymbolKeyDescriptor(
     /** If `true`, original values are saved; if `false`, only hashes are saved. */
     private val storeFullFqNames: Boolean = false
@@ -278,51 +261,6 @@ class DelegateDataExternalizer<T>(
     }
 }
 
-/**
- * [DataExternalizer] for a [Collection].
- *
- * If you need a [DataExternalizer] for a more specific instance of [Collection] (e.g., [List]), use [ListExternalizer] or create another
- * instance of [GenericCollectionExternalizer].
- *
- * Note: The implementations of this class and [GenericCollectionExternalizer] are similar but not exactly the same: the latter reads and
- * writes the size of the collection to avoid resizing the collection when reading. Therefore, if we make this class extend
- * [GenericCollectionExternalizer] to share code, we will need to update some expected files in tests as the serialized data will change
- * slightly.
- */
-open class CollectionExternalizer<T>(
-    private val elementExternalizer: DataExternalizer<T>,
-    private val newCollection: () -> MutableCollection<T>
-) : AppendableDataExternalizer<Collection<T>> {
-    override fun read(input: DataInput): Collection<T> {
-        val result = newCollection()
-        val stream = input as DataInputStream
-
-        while (stream.available() > 0) {
-            result.add(elementExternalizer.read(stream))
-        }
-
-        return result
-    }
-
-    override fun save(output: DataOutput, value: Collection<T>) {
-        value.forEach { elementExternalizer.save(output, it) }
-    }
-
-    override fun createNil() = newCollection()
-
-    override fun append(currentValue: Collection<T>, appendData: Collection<T>) = when (currentValue) {
-        is MutableCollection<*> -> {
-            (currentValue as MutableCollection<T>).addAll(appendData)
-            currentValue
-        }
-        else -> currentValue + appendData
-    }
-}
-
-object StringCollectionExternalizer : CollectionExternalizer<String>(EnumeratorStringDescriptor(), { HashSet() })
-
-object IntCollectionExternalizer : CollectionExternalizer<Int>(IntExternalizer, { HashSet() })
-
 fun DataOutput.writeString(value: String) = StringExternalizer.save(this, value)
 
 fun DataInput.readString(): String = StringExternalizer.read(this)
@@ -358,9 +296,56 @@ object ByteArrayExternalizer : DataExternalizer<ByteArray> {
     }
 }
 
-abstract class GenericCollectionExternalizer<T, C : Collection<T>>(
+/**
+ * DEPRECATED: [DataExternalizer] for a [Collection], whose implementation is tied to [com.intellij.util.io.PersistentHashMap] (e.g., the
+ * [read] method reads until the stream ends -- this can only work with a [com.intellij.util.io.PersistentHashMap]).
+ *
+ * Use [CollectionExternalizerV2] if possible.
+ */
+private class CollectionExternalizerForPersistentHashMap<T>(
     private val elementExternalizer: DataExternalizer<T>,
-    private val newCollection: (size: Int) -> MutableCollection<T>
+    private val newCollection: () -> MutableCollection<T>,
+) : DataExternalizer<Collection<T>> {
+
+    override fun save(output: DataOutput, value: Collection<T>) {
+        value.forEach { elementExternalizer.save(output, it) }
+    }
+
+    override fun read(input: DataInput): Collection<T> {
+        val result = newCollection()
+        val stream = input as DataInputStream
+
+        while (stream.available() > 0) {
+            result.add(elementExternalizer.read(stream))
+        }
+
+        return result
+    }
+}
+
+/**
+ * DEPRECATED: This class should not be used because its implementation is tied to [com.intellij.util.io.PersistentHashMap]
+ * (see [CollectionExternalizerForPersistentHashMap]).
+ *
+ * Currently, we can't change the name or implementation of this class because it is still used by the `compiler-reference-index` module in
+ * the Kotlin IDEA plugin and that code relies on this name and implementation being unchanged (see KT-62288).
+ *
+ * Once we remove that dependency, we can remove this class.
+ */
+class CollectionExternalizer<T>(
+    private val elementExternalizer: DataExternalizer<T>,
+    private val newCollection: () -> MutableCollection<T>,
+) : DataExternalizer<Collection<T>> by CollectionExternalizerForPersistentHashMap(elementExternalizer, newCollection)
+
+/** DEPRECATED: See [CollectionExternalizer]. */
+@Suppress("unused") // See `CollectionExternalizer`
+object IntCollectionExternalizer :
+    DataExternalizer<Collection<Int>> by CollectionExternalizerForPersistentHashMap(IntExternalizer, { ArrayList() })
+
+/** [DataExternalizer] for a [Collection]. */
+open class CollectionExternalizerV2<T, C : Collection<T>>(
+    private val elementExternalizer: DataExternalizer<T>,
+    private val newCollection: (size: Int) -> MutableCollection<T>,
 ) : DataExternalizer<C> {
 
     override fun save(output: DataOutput, collection: C) {
@@ -384,11 +369,13 @@ abstract class GenericCollectionExternalizer<T, C : Collection<T>>(
     }
 }
 
+object StringCollectionExternalizer : CollectionExternalizerV2<String, Collection<String>>(StringExternalizer, { size -> ArrayList(size) })
+
 class ListExternalizer<T>(elementExternalizer: DataExternalizer<T>) :
-    GenericCollectionExternalizer<T, List<T>>(elementExternalizer, { size -> ArrayList(size) })
+    CollectionExternalizerV2<T, List<T>>(elementExternalizer, { size -> ArrayList(size) })
 
 class SetExternalizer<T>(elementExternalizer: DataExternalizer<T>) :
-    GenericCollectionExternalizer<T, Set<T>>(elementExternalizer, { size -> LinkedHashSet(size) })
+    CollectionExternalizerV2<T, Set<T>>(elementExternalizer, { size -> LinkedHashSet(size) })
 
 open class MapExternalizer<K, V, M : Map<K, V>>(
     private val keyExternalizer: DataExternalizer<K>,

build-common/test/org/jetbrains/kotlin/incremental/CompilationTransactionTest.kt

@@ -6,14 +6,13 @@
 package org.jetbrains.kotlin.incremental
 
 import org.jetbrains.kotlin.build.report.DoNothingBuildReporter
-import org.jetbrains.kotlin.incremental.storage.InMemoryStorageWrapper
+import org.jetbrains.kotlin.incremental.storage.InMemoryStorageInterface
-import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Assertions.*
-import org.junit.jupiter.api.Assertions.assertFalse
-import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.assertThrows
 import org.junit.jupiter.api.io.TempDir
 import java.io.Closeable
+import java.io.File
 import java.nio.file.Files
 import java.nio.file.Path
 
@@ -27,23 +26,21 @@ private class CacheMock(private val throwsException: Boolean = false) : Closeabl
     }
 }
 
-private class InMemoryStorageWrapperMock : InMemoryStorageWrapper<Any, Any> {
+private class InMemoryStorageWrapperMock : InMemoryStorageInterface<Any, Any> {
     var reset = false
 
-    override fun resetInMemoryChanges() {
+    override fun applyChanges() {}
+
+    override fun clearChanges() {
         reset = true
     }
 
-    override val keys: Collection<Any> = emptyList()
+    override val storageFile = File("")
 
-    override fun clean() {}
+    override val keys: Set<Any> = emptySet()
-
-    override fun flush(memoryCachesOnly: Boolean) {}
 
     override fun close() {}
 
-    override fun append(key: Any, value: Any) {}
-
     override fun remove(key: Any) {}
 
     override fun set(key: Any, value: Any) {}

build-common/test/org/jetbrains/kotlin/incremental/storage/InMemoryStorageTest.kt

@@ -15,7 +15,7 @@ import java.io.Closeable
 import java.nio.file.Files
 import java.nio.file.Path
 
-class InMemoryStorageWrapperTest {
+class InMemoryStorageTest {
     @TempDir
     private lateinit var workingDir: Path
 
@@ -83,7 +83,7 @@ class InMemoryStorageWrapperTest {
             assertEquals(setOf(5), it[key5])
         }
         withLookupMapInTransaction(storageRoot, useInMemoryWrapper = true, successful = true) {
-            it.clean()
+            it.clear()
             it.append(key1, setOf(4))
         }
         withLookupMapInTransaction(storageRoot, useInMemoryWrapper = false, successful = true) {
@@ -108,7 +108,7 @@ class InMemoryStorageWrapperTest {
         val savedState = workingDir.resolve("backup")
         storageRoot.toFile().copyRecursively(savedState.toFile())
         withLookupMapInTransaction(storageRoot, useInMemoryWrapper = true, successful = false) {
-            it.clean()
+            it.clear()
             it.append(key1, setOf(3))
             it.remove(key2)
             it[key3] = setOf(5)
@@ -131,7 +131,6 @@ class InMemoryStorageWrapperTest {
         icContext.transaction.runWithin { transaction ->
             val lookupMap = LookupMap(storageFile, icContext)
             transaction.cachesManager = Closeable {
-                lookupMap.flush(false)
                 lookupMap.close()
             }
             dataProvider(lookupMap)

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/IncrementalCachesManager.kt

@@ -60,8 +60,6 @@ abstract class IncrementalCachesManager<PlatformCache : AbstractIncrementalCache
     private class CacheCloser(private val cache: BasicMapsOwner) : Closeable {
 
         override fun close() {
-            // It's important to flush the cache when closing (see KT-53168)
-            cache.flush(memoryCachesOnly = false)
             cache.close()
         }
     }

compiler/incremental-compilation-impl/src/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMap.kt

@@ -14,7 +14,8 @@ class SourceToOutputFilesMap(
     icContext: IncrementalCompilationContext,
 ) : BasicStringMap<Collection<String>>(storageFile, PathStringDescriptor, StringCollectionExternalizer, icContext) {
     operator fun set(sourceFile: File, outputFiles: Collection<File>) {
-        storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] = outputFiles.map(icContext.pathConverterForOutputFiles::toPath)
+        storage[icContext.pathConverterForSourceFiles.toPath(sourceFile)] =
+            outputFiles.toSet().map(icContext.pathConverterForOutputFiles::toPath)
     }
 
     operator fun get(sourceFile: File): Collection<File>? =

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/snapshots/FileSnapshotMapTest.kt

@@ -32,8 +32,7 @@ class FileSnapshotMapTest : TestWithWorkingDir() {
 
     @After
     override fun tearDown() {
-        snapshotMap.flush(false)
+        snapshotMap.close()
-        snapshotMap.closeForTest()
         super.tearDown()
     }
 

compiler/incremental-compilation-impl/test/org/jetbrains/kotlin/incremental/storage/SourceToOutputFilesMapTest.kt

@@ -50,8 +50,7 @@ class SourceToOutputFilesMapTest {
 
     @After
     fun tearDown() {
-        stofMap.flush(false)
+        stofMap.close()
-        stofMap.closeForTest()
     }
 
     @Test