[JS IR] IC invalidation performance improvements

- Cache signature readers and deserializers
 - Cache inline function hashes
 - IC refactoring
 - [gradle] Use hash of module path for cache dir; Fix KT-51238

^KT-51238 Fixed

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -43,9 +43,7 @@ import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer
 import org.jetbrains.kotlin.backend.wasm.dce.eliminateDeadDeclarations
 import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
-import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCaches
+import org.jetbrains.kotlin.ir.backend.js.ic.*
-import org.jetbrains.kotlin.ir.backend.js.ic.buildCacheForModuleFiles
-import org.jetbrains.kotlin.ir.backend.js.ic.loadModuleCaches
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformerTmp
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.SourceMapsInfo
@@ -210,7 +208,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
 
             val includes = arguments.includes!!
 
-            val start = System.currentTimeMillis()
+            var start = System.currentTimeMillis()
 
             actualizeCaches(
                 includes,
@@ -220,12 +218,15 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
                 { IrFactoryImplForJsIC(WholeWorldStageController()) },
                 mainCallArguments,
                 ::buildCacheForModuleFiles,
-            ) { updateStatus ->
+            ) { updateStatus, updatedModule ->
-                if (updateStatus.upToDate) {
+                val now = System.currentTimeMillis()
-                    messageCollector.report(INFO, "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms")
+                val strStatus = when (updateStatus) {
-                } else {
+                    CacheUpdateStatus.FAST_PATH -> "up-to-date; fast check"
-                    messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms")
+                    CacheUpdateStatus.NO_DIRTY_FILES -> "up-to-date; full check"
+                    CacheUpdateStatus.DIRTY -> "dirty; cache building"
                 }
+                messageCollector.report(INFO, "IC per-file is $strStatus duration ${now - start}ms; module [${File(updatedModule).name}]")
+                start = now
             }
         } else emptyList()
 

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheInfo.kt

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import java.io.File
+
+// TODO md5 hash
+data class CacheInfo(
+    val path: String,
+    val libPath: String,
+    val moduleName: String?,
+    var flatHash: ULong,
+    var transHash: ULong,
+    var configHash: ULong
+) {
+    companion object {
+        fun load(path: String): CacheInfo? {
+            val info = File(File(path), "info")
+
+            if (!info.exists()) return null
+
+            val (libPath, moduleName, flatHash, transHash, configHash) = info.readLines()
+
+            // safe cast for the backward compatibility with the cache from the previous compiler versions
+            val configHashULong = configHash.toULongOrNull(16) ?: 0UL
+            return CacheInfo(path, libPath, moduleName, flatHash.toULong(16), transHash.toULong(16), configHashULong)
+        }
+
+        fun loadOrCreate(path: String, libPath: String): CacheInfo {
+            return load(path) ?: CacheInfo(path, libPath, null, 0UL, 0UL, 0UL)
+        }
+    }
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheLazyLoader.kt

@@ -0,0 +1,86 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
+import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
+import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
+import org.jetbrains.kotlin.ir.util.IdSignature
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
+
+
+class CacheLazyLoader(private val cacheProvider: PersistentCacheProvider, private val library: KotlinLibrary) {
+    private val graphInlineCache = mutableMapOf<String, Collection<Pair<IdSignature, TransHash>>>()
+    private val fingerPrintCache = mutableMapOf<String, Hash>()
+
+    val signatureReadersList: List<Pair<String, IdSignatureDeserializer>> by lazy {
+        library.filesAndSigReaders()
+    }
+
+    private val signatureReadersMap: Map<String, IdSignatureDeserializer> by lazy {
+        signatureReadersList.toMap()
+    }
+
+    val allInlineFunctionHashes: Map<IdSignature, TransHash> by lazy {
+        cacheProvider.allInlineHashes { librarySrc, index ->
+            val libReader = signatureReadersMap[librarySrc] ?: error("No module reader for lib $librarySrc")
+            libReader.deserializeIdSignature(index)
+        }
+    }
+
+    fun getInlineHashesByFile() = signatureReadersList.associateTo(mutableMapOf()) {
+        it.first to cacheProvider.inlineHashes(it.first) { index -> it.second.deserializeIdSignature(index) }
+    }
+
+    fun getInlineGraphForFile(srcFile: String) = graphInlineCache.getOrPut(srcFile) {
+        val fileReader = signatureReadersMap[srcFile] ?: error("Cannot find signature reader for $srcFile")
+        cacheProvider.inlineGraphForFile(srcFile) { index -> fileReader.deserializeIdSignature(index) }
+    }
+
+    fun getFileFingerPrint(srcFile: String) = fingerPrintCache.getOrPut(srcFile) {
+        cacheProvider.fileFingerPrint(srcFile)
+    }
+
+    fun getFilePaths() = cacheProvider.filePaths()
+
+    fun clearCaches() {
+        graphInlineCache.clear()
+        fingerPrintCache.clear()
+    }
+
+    private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDeserializer>> {
+        val fileSize = fileCount()
+        val result = ArrayList<Pair<String, IdSignatureDeserializer>>(fileSize)
+        val extReg = ExtensionRegistryLite.newInstance()
+
+        for (i in 0 until fileSize) {
+            val fileStream = file(i).codedInputStream
+            val fileProto = IrFile.parseFrom(fileStream, extReg)
+            val sigReader = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
+                private fun err(): Nothing = error("Not supported")
+                override fun irDeclaration(index: Int): ByteArray = err()
+
+                override fun type(index: Int): ByteArray = err()
+
+                override fun signature(index: Int): ByteArray = signature(index, i)
+
+                override fun string(index: Int): ByteArray = string(index, i)
+
+                override fun body(index: Int): ByteArray = err()
+
+                override fun debugInfo(index: Int): ByteArray? = null
+            }), null)
+
+            result.add(fileProto.fileEntry.name to sigReader)
+        }
+
+        return result
+    }
+}
+

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt

@@ -1,45 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.ic
-
-import java.io.File
-import java.io.PrintWriter
-
-// TODO md5 hash
-data class CacheInfo(val path: String, val libPath: String, var flatHash: ULong, var transHash: ULong, var configHash: ULong) {
-    fun save() {
-        PrintWriter(File(File(path), "info")).use {
-            it.println(libPath)
-            it.println(flatHash.toString(16))
-            it.println(transHash.toString(16))
-            it.println(configHash.toString(16))
-        }
-    }
-
-    companion object {
-        fun load(path: String): CacheInfo? {
-            val info = File(File(path), "info")
-
-            if (!info.exists()) return null
-
-            val (libPath, _, flatHash, transHash, configHash) = info.readLines()
-
-            // safe cast for the backward compatibility with the cache from the previous compiler versions
-            val configHashULong = configHash.toULongOrNull(16) ?: 0UL
-            return CacheInfo(path, libPath, flatHash.toULong(16), transHash.toULong(16), configHashULong)
-        }
-
-        fun loadOrCreate(
-            path: String,
-            moduleName: String,
-            flatHash: ULong = 0UL,
-            transHash: ULong = 0UL,
-            configHash: ULong = 0UL
-        ): CacheInfo {
-            return load(path) ?: CacheInfo(path, moduleName, flatHash, transHash, configHash)
-        }
-    }
-}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt

@@ -5,10 +5,6 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
-import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
-import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
 import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.LanguageVersionSettings
@@ -32,12 +28,10 @@ import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.unresolvedDependencies
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import java.io.File
 import java.security.MessageDigest
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
 
 private fun KotlinLibrary.fingerprint(fileIndex: Int): Hash {
     return ((((types(fileIndex).md5() * 31) + signatures(fileIndex).md5()) * 31 + strings(fileIndex).md5()) * 31 + declarations(fileIndex).md5()) * 31 + bodies(
@@ -50,11 +44,9 @@ private fun invalidateCacheForModule(
     libraryFiles: List<String>,
     externalHashes: Map<IdSignature, TransHash>,
     cachedInlineHashesForFile: MutableMap<String, Map<IdSignature, TransHash>>,
-    cacheProvider: PersistentCacheProvider,
+    cacheProvider: CacheLazyLoader,
     cacheConsumer: PersistentCacheConsumer,
-    signatureResolver: (String, Int) -> IdSignature,
+    fileFingerPrints: MutableMap<String, Hash>
-    fileFingerPrints: MutableMap<String, Hash>,
-    configUpdated: Boolean
 ): Pair<Set<String>, Collection<String>> {
 
     val dirtyFiles = mutableSetOf<String>()
@@ -62,12 +54,12 @@ private fun invalidateCacheForModule(
     for ((index, file) in libraryFiles.withIndex()) {
 
         // 1. get cached fingerprints
-        val fileOldFingerprint = cacheProvider.fileFingerPrint(file)
+        val fileOldFingerprint = cacheProvider.getFileFingerPrint(file)
 
         // 2. calculate new fingerprints
         val fileNewFingerprint = library.fingerprint(index)
 
-        if (fileOldFingerprint != fileNewFingerprint || configUpdated) {
+        if (fileOldFingerprint != fileNewFingerprint) {
             fileFingerPrints[file] = fileNewFingerprint
             cachedInlineHashesForFile.remove(file)
 
@@ -77,33 +69,26 @@ private fun invalidateCacheForModule(
     }
 
     // 4. extend dirty set with inline functions
-
-    val graphCache = mutableMapOf<FilePath, Collection<Pair<IdSignature, TransHash>>>()
-
-    var oldSize: Int
     do {
-        oldSize = dirtyFiles.size
-
         if (dirtyFiles.size == libraryFiles.size) break
 
+        val oldSize = dirtyFiles.size
         for (file in libraryFiles) {
 
             if (file in dirtyFiles) continue
 
             // check for clean file
-            val inlineGraph = graphCache.getOrPut(file) { cacheProvider.inlineGraphForFile(file) { signatureResolver(file, it) } }
+            val inlineGraph = cacheProvider.getInlineGraphForFile(file)
 
             for ((sig, oldHash) in inlineGraph) {
                 val actualHash = externalHashes[sig] ?: cachedInlineHashesForFile.values.firstNotNullOfOrNull { it[sig] }
-
                 // null means inline function is from dirty file, could be a bit more optimal
-                if (actualHash != null) {
+                if (actualHash == null || oldHash != actualHash) {
-                    if (oldHash == actualHash) continue
+                    cachedInlineHashesForFile.remove(file)
+                    dirtyFiles.add(file)
+                    fileFingerPrints[file] = cacheProvider.getFileFingerPrint(file)
+                    break
                 }
-
-                cachedInlineHashesForFile.remove(file)
-                dirtyFiles.add(file)
-                fileFingerPrints[file] = cacheProvider.fileFingerPrint(file)
             }
         }
     } while (oldSize != dirtyFiles.size)
@@ -113,47 +98,28 @@ private fun invalidateCacheForModule(
         cacheConsumer.invalidateForFile(dirty)
     }
 
-    val cachedFiles = cacheProvider.filePaths()
+    val cachedFiles = cacheProvider.getFilePaths()
     val deletedFiles = cachedFiles - libraryFiles.toSet()
 
     for (deleted in deletedFiles) {
         cacheConsumer.invalidateForFile(deleted)
     }
 
+    cacheProvider.clearCaches()
     return dirtyFiles to deletedFiles
 }
 
-private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDeserializer>> {
+private fun CacheInfo.commitLibraryInfo(cacheConsumer: PersistentCacheConsumer, newModuleName: String? = null) {
-    val fileSize = fileCount()
+    cacheConsumer.commitLibraryInfo(
-    val result = ArrayList<Pair<String, IdSignatureDeserializer>>(fileSize)
+        libPath.toCanonicalPath(),
-    val extReg = ExtensionRegistryLite.newInstance()
+        newModuleName ?: moduleName ?: error("Cannot find module name for $libPath module"),
-
+        flatHash,
-    for (i in 0 until fileSize) {
+        transHash,
-        val fileStream = file(i).codedInputStream
+        configHash
-        val fileProto = ProtoFile.parseFrom(fileStream, extReg)
+    )
-        val sigReader = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
-            private fun err(): Nothing = error("Not supported")
-            override fun irDeclaration(index: Int): ByteArray = err()
-
-            override fun type(index: Int): ByteArray = err()
-
-            override fun signature(index: Int): ByteArray = signature(index, i)
-
-            override fun string(index: Int): ByteArray = string(index, i)
-
-            override fun body(index: Int): ByteArray = err()
-
-            override fun debugInfo(index: Int): ByteArray? = null
-        }), null)
-
-        result.add(fileProto.fileEntry.name to sigReader)
-    }
-
-    return result
 }
 
 private fun buildCacheForModule(
-    libraryInfo: CacheInfo,
     configuration: CompilerConfiguration,
     irModule: IrModuleFragment,
     deserializer: JsIrLinker,
@@ -217,14 +183,6 @@ private fun buildCacheForModule(
         emptySet(),
         mainArguments
     )
-
-    cacheConsumer.commitLibraryInfo(
-        libraryInfo.libPath.toCanonicalPath(),
-        irModule.name.asString(),
-        libraryInfo.flatHash,
-        libraryInfo.transHash,
-        libraryInfo.configHash
-    )
 }
 
 private fun loadModules(
@@ -303,6 +261,10 @@ fun loadCacheInfo(cachePaths: Collection<String>): MutableMap<ModulePath, CacheI
     return caches.associateByTo(result) { it.libPath.toCanonicalPath() }
 }
 
+fun String.toCanonicalPath(): String = File(this).canonicalPath
+
+private fun KotlinLibrary.moduleCanonicalName() = libraryFile.path.toCanonicalPath()
+
 private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Collection<String>): Map<ModulePath, KotlinLibrary> {
     val allResolvedDependencies = jsResolveLibraries(
         dependencies,
@@ -310,11 +272,9 @@ private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Co
         configuration[IrMessageLogger.IR_MESSAGE_LOGGER].toResolverLogger()
     )
 
-    return allResolvedDependencies.getFullList().associateBy { it.libraryFile.path.toCanonicalPath() }
+    return allResolvedDependencies.getFullList().associateBy { it.moduleCanonicalName() }
 }
 
-fun String.toCanonicalPath(): String = File(this).canonicalPath
-
 typealias ModuleName = String
 typealias ModulePath = String
 typealias FilePath = String
@@ -375,22 +335,20 @@ private fun CompilerConfiguration.calcMD5(): ULong {
 }
 
 private fun checkLibrariesHash(
-    libraries: Map<ModulePath, KotlinLibrary>,
+    currentLib: KotlinLibrary,
     dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
+    currentCache: CacheInfo,
     icCacheMap: Map<ModulePath, CacheInfo>,
     modulePath: ModulePath
 ): Boolean {
-    val currentLib = libraries[modulePath] ?: error("1")
-    val currentCache = icCacheMap[modulePath] ?: error("2")
-
     val flatHash = File(modulePath).md5()
 
-    val dependencies = dependencyGraph[currentLib] ?: error("3")
+    val dependencies = dependencyGraph[currentLib] ?: error("Cannot find dependencies for ${currentLib.libraryName}")
 
     var transHash = flatHash
 
     for (dep in dependencies) {
-        val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("4")
+        val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("Cannot cache info for ${dep.libraryName}")
         transHash += depCache.transHash
     }
 
@@ -403,11 +361,19 @@ private fun checkLibrariesHash(
     return true
 }
 
+private fun CacheInfo.invalidateCacheForNewConfig(configMD5: ULong, cacheConsumer: PersistentCacheConsumer) {
+    if (configHash != configMD5) {
+        cacheConsumer.invalidate()
+        flatHash = 0UL
+        transHash = 0UL
+        configHash = configMD5
+    }
+}
+
 enum class CacheUpdateStatus(val upToDate: Boolean) {
     DIRTY(upToDate = false),
     NO_DIRTY_FILES(upToDate = true),
     FAST_PATH(upToDate = true)
-
 }
 
 fun actualizeCaches(
@@ -418,13 +384,14 @@ fun actualizeCaches(
     irFactory: () -> IrFactory,
     mainArguments: List<String>?,
     executor: CacheExecutor,
-    callback: (CacheUpdateStatus) -> Unit
+    callback: (CacheUpdateStatus, String) -> Unit
 ): List<String> {
     val (libraries, dependencyGraph, configMD5) = CacheConfiguration(dependencies, compilerConfiguration)
     val cacheMap = libraries.values.zip(icCachePaths).toMap()
 
-    val icCacheMap: MutableMap<ModulePath, CacheInfo> = mutableMapOf<ModulePath, CacheInfo>()
+    val icCacheMap = mutableMapOf<ModulePath, CacheInfo>()
     val resultCaches = mutableListOf<String>()
+    val persistentCacheProviders = mutableMapOf<KotlinLibrary, CacheLazyLoader>()
 
     val visitedLibraries = mutableSetOf<KotlinLibrary>()
     fun visitDependency(library: KotlinLibrary) {
@@ -437,23 +404,30 @@ fun actualizeCaches(
         val cachePath = cacheMap[library] ?: error("Unknown cache for library ${library.libraryName}")
         resultCaches.add(cachePath)
 
-        val moduleName = library.libraryFile.path.toCanonicalPath()
+        val modulePath = library.moduleCanonicalName()
-        val cacheInfo = CacheInfo.loadOrCreate(cachePath, moduleName, configHash = configMD5)
+        val cacheInfo = CacheInfo.loadOrCreate(cachePath, modulePath)
-        icCacheMap[moduleName] = cacheInfo
+        icCacheMap[modulePath] = cacheInfo
 
-        val updateStatus = actualizeCacheForModule(
+        val cacheConsumer = createCacheConsumer(cachePath)
-            moduleName = moduleName,
+        persistentCacheProviders[library] = CacheLazyLoader(createCacheProvider(cachePath), library)
-            cachePath = cachePath,
+
-            compilerConfiguration = compilerConfiguration,
+        cacheInfo.invalidateCacheForNewConfig(configMD5, cacheConsumer)
-            configMD5 = configMD5,
+        val updateStatus = when {
-            libraries = libraries,
+            checkLibrariesHash(library, dependencyGraph, cacheInfo, icCacheMap, modulePath) -> CacheUpdateStatus.FAST_PATH
-            dependencyGraph = dependencyGraph,
+            else -> actualizeCacheForModule(
-            icCacheMap = icCacheMap,
+                library = library,
-            irFactory = irFactory(),
+                libraryInfo = cacheInfo,
-            mainArguments = mainArguments,
+                configuration = compilerConfiguration,
-            executor = executor
+                dependencyGraph = getDependencySubGraphFor(library, dependencyGraph),
-        )
+                persistentCacheProviders = persistentCacheProviders,
-        callback(updateStatus)
+                persistentCacheConsumer = cacheConsumer,
+                irFactory = irFactory(),
+                mainArguments = mainArguments,
+                cacheExecutor = executor,
+            )
+        }
+
+        callback(updateStatus, modulePath)
     }
 
     val canonicalIncludes = includes.toCanonicalPath()
@@ -462,6 +436,26 @@ fun actualizeCaches(
     return resultCaches
 }
 
+private fun getDependencySubGraphFor(
+    targetLib: KotlinLibrary,
+    dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>
+): Map<KotlinLibrary, List<KotlinLibrary>> {
+    val subGraph = mutableMapOf<KotlinLibrary, List<KotlinLibrary>>()
+
+    fun addDependsFor(library: KotlinLibrary) {
+        if (library in subGraph) {
+            return
+        }
+        val dependencies = dependencyGraph[library] ?: error("Cannot find dependencies for ${library.libraryName}")
+        subGraph[library] = dependencies
+        for (dependency in dependencies) {
+            addDependsFor(dependency)
+        }
+    }
+    addDependsFor(targetLib)
+    return subGraph
+}
+
 class CacheConfiguration(
     private val dependencies: Collection<ModulePath>,
     val compilerConfiguration: CompilerConfiguration
@@ -487,97 +481,32 @@ class CacheConfiguration(
     operator fun component3() = configMD5
 }
 
-// Returns true if caches up-to-date
-fun actualizeCacheForModule(
-    moduleName: String,
-    cachePath: String,
-    compilerConfiguration: CompilerConfiguration,
-    configMD5: ULong,
-    libraries: Map<ModulePath, KotlinLibrary>,
-    dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
-    icCacheMap: Map<ModulePath, CacheInfo>,
-    irFactory: IrFactory,
-    mainArguments: List<String>?,
-    executor: CacheExecutor
-): CacheUpdateStatus {
-    val cacheInfo = icCacheMap[moduleName] ?: error("Cache for $moduleName not found")
-
-    val configUpdated = configMD5 != cacheInfo.configHash
-    cacheInfo.configHash = configMD5
-    if (checkLibrariesHash(libraries, dependencyGraph, icCacheMap, moduleName) && !configUpdated) {
-        return CacheUpdateStatus.FAST_PATH // up-to-date
-    }
-
-    val persistentCacheProviders = icCacheMap.map { (lib, cache) ->
-        libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache.path)
-    }.toMap()
-
-    val currentModule = libraries[moduleName] ?: error("No loaded library found for path $moduleName")
-    val persistentCacheConsumer = createCacheConsumer(cachePath)
-
-    return actualizeCacheForModule(
-        currentModule,
-        cacheInfo,
-        compilerConfiguration,
-        dependencyGraph,
-        persistentCacheProviders,
-        persistentCacheConsumer,
-        irFactory,
-        mainArguments,
-        executor,
-        configUpdated
-    )
-}
-
-
 private fun actualizeCacheForModule(
     library: KotlinLibrary,
     libraryInfo: CacheInfo,
     configuration: CompilerConfiguration,
     dependencyGraph: Map<KotlinLibrary, Collection<KotlinLibrary>>,
-    persistentCacheProviders: Map<KotlinLibrary, PersistentCacheProvider>,
+    persistentCacheProviders: Map<KotlinLibrary, CacheLazyLoader>,
     persistentCacheConsumer: PersistentCacheConsumer,
     irFactory: IrFactory,
     mainArguments: List<String>?,
     cacheExecutor: CacheExecutor,
-    configUpdated: Boolean
 ): CacheUpdateStatus {
     // 1. Invalidate
     val dependencies = dependencyGraph[library]!!
 
-    val filesAndSigReaders = library.filesAndSigReaders()
+    val currentLibraryCacheProvider = persistentCacheProviders[library] ?: error("No cache provider for $library")
-    val signatureDeserializers = filesAndSigReaders.toMap()
+    val libraryFiles = currentLibraryCacheProvider.signatureReadersList.map { it.first }
-    val libraryFiles = filesAndSigReaders.map { it.first }
-
-    val depReaders = dependencies.associateWith { it.filesAndSigReaders().toMap() }
-
-    val signatureResolver: (String, Int) -> IdSignature = { f, s ->
-        signatureDeserializers[f]?.deserializeIdSignature(s) ?: error("Cannot deserialize sig $s from $f")
-    }
 
     val sigHashes = mutableMapOf<IdSignature, TransHash>()
     dependencies.forEach { lib ->
         persistentCacheProviders[lib]?.let { provider ->
-            val moduleReaders = depReaders[lib]!!
+            sigHashes.putAll(provider.allInlineFunctionHashes)
-            val inlineHashes = provider.allInlineHashes { f, i ->
-                val moduleReader = moduleReaders[f]
-                    ?: error("No module reader for file $f")
-                moduleReader.deserializeIdSignature(i)
-            }
-            sigHashes.putAll(inlineHashes)
         }
     }
 
     val fileFingerPrints = mutableMapOf<String, Hash>()
-
+    val fileCachedInlineHashes = currentLibraryCacheProvider.getInlineHashesByFile()
-    val currentLibraryCacheProvider = persistentCacheProviders[library] ?: error("No cache provider for $library")
-
-    val fileCachedInlineHashes = mutableMapOf<String, Map<IdSignature, TransHash>>()
-
-    filesAndSigReaders.forEach { (filePath, sigReader) ->
-        fileCachedInlineHashes[filePath] =
-            currentLibraryCacheProvider.inlineHashes(filePath) { s -> sigReader.deserializeIdSignature(s) }
-    }
 
     val (dirtySet, deletedFiles) = invalidateCacheForModule(
         library,
@@ -586,12 +515,14 @@ private fun actualizeCacheForModule(
         fileCachedInlineHashes,
         currentLibraryCacheProvider,
         persistentCacheConsumer,
-        signatureResolver,
+        fileFingerPrints
-        fileFingerPrints,
-        configUpdated
     )
 
-    if (dirtySet.isEmpty()) return CacheUpdateStatus.NO_DIRTY_FILES // up-to-date
+    if (dirtySet.isEmpty()) {
+        // up-to-date
+        libraryInfo.commitLibraryInfo(persistentCacheConsumer)
+        return CacheUpdateStatus.NO_DIRTY_FILES
+    }
 
     // 2. Build
 
@@ -619,17 +550,13 @@ private fun actualizeCacheForModule(
     val currentIrModule = irModules.find { it.second == library }?.first!!
     val currentModuleDeserializer = jsIrLinker.moduleDeserializer(currentIrModule.descriptor)
 
-    for (file in libraryFiles) {
+    for (hashes in fileCachedInlineHashes.values) {
-        if (file !in dirtySet) {
+        sigHashes.putAll(hashes)
-            val sigDeserializer = signatureDeserializers[file]!!
-            sigHashes.putAll(currentLibraryCacheProvider.inlineHashes(file) { sigDeserializer.deserializeIdSignature(it) })
-        }
     }
 
     val deserializers = dirtySet.associateWith { currentModuleDeserializer.signatureDeserializerForFile(it).signatureToIndexMapping() }
 
     buildCacheForModule(
-        libraryInfo,
         configuration,
         currentIrModule,
         jsIrLinker,
@@ -643,6 +570,8 @@ private fun actualizeCacheForModule(
         mainArguments,
         cacheExecutor
     )
+
+    libraryInfo.commitLibraryInfo(persistentCacheConsumer, currentIrModule.name.asString())
     return CacheUpdateStatus.DIRTY // invalidated and re-built
 }
 
@@ -684,7 +613,7 @@ fun rebuildCacheForDirtyFiles(
 
     val currentIrModule = irModules.find { it.second == library }?.first!!
 
-    cacheConsumer.commitLibraryInfo(library.libraryFile.path.toCanonicalPath(), currentIrModule.name.asString(), 0UL, 0UL, 0UL)
+    cacheConsumer.commitLibraryInfo(library.moduleCanonicalName(), currentIrModule.name.asString(), 0UL, 0UL, 0UL)
 
     buildCacheForModuleFiles(
         currentIrModule,
@@ -739,4 +668,4 @@ fun loadModuleCaches(icCachePaths: Collection<String>): Map<String, ModuleCache>
             f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f))
         })
     }
-}
+}

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt

@@ -202,6 +202,7 @@ interface PersistentCacheConsumer {
     fun commitBinaryDts(path: String, dstData: ByteArray)
     fun commitSourceMap(path: String, mapData: ByteArray)
     fun invalidateForFile(path: String)
+    fun invalidate()
 
     fun commitLibraryInfo(libraryPath: String, moduleName: String, flatHash: ULong, transHash: ULong, configHash: ULong)
 
@@ -227,6 +228,9 @@ interface PersistentCacheConsumer {
             override fun invalidateForFile(path: String) {
             }
 
+            override fun invalidate() {
+            }
+
             override fun commitBinaryAst(path: String, astData: ByteArray) {
 
             }
@@ -303,6 +307,10 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac
         //fileDir.deleteRecursively()
     }
 
+    override fun invalidate() {
+        File(cachePath).deleteRecursively()
+    }
+
     private fun commitByteArrayToCacheFile(fileName: String, cacheName: String, data: ByteArray) {
         val fileId = createFileCacheId(fileName)
         val cacheDir = File(File(cachePath), fileId)

js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt

@@ -254,29 +254,25 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
         mainArguments: List<String>?,
         executor: CacheExecutor
     ): CacheUpdateStatus {
-        val (libraries, dependencyGraph, configMD5) = CacheConfiguration(dependencies, compilerConfiguration)
-
         val modulePath = moduleName.toCanonicalPath()
+
         val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths).also {
-            it[modulePath] = CacheInfo.loadOrCreate(
+            it[modulePath] = CacheInfo.loadOrCreate(cachePath, modulePath)
-                cachePath,
-                moduleName,
-                configHash = configMD5
-            )
         }
 
-        return actualizeCacheForModule(
+        val statuses = mutableMapOf<String, CacheUpdateStatus>()
-            moduleName = modulePath,
+        actualizeCaches(
-            cachePath = cachePath,
+            modulePath,
-            compilerConfiguration = compilerConfiguration,
+            compilerConfiguration,
-            configMD5 = configMD5,
+            dependencies,
-            libraries = libraries,
+            icCacheMap.map { it.value.path },
-            dependencyGraph = dependencyGraph,
+            { irFactory },
-            icCacheMap = icCacheMap,
+            mainArguments,
-            irFactory = irFactory,
+            executor
-            mainArguments = mainArguments,
+        ) { updateStatus, updatedModule ->
-            executor = executor
+            statuses[updatedModule] = updateStatus
-        )
+        }
+        return statuses[modulePath] ?: error("Status is missed for $modulePath")
     }
 
     private fun KotlinCoreEnvironment.createPsiFile(file: File): KtFile {
@@ -347,4 +343,4 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
         const val TEST_DATA_DIR_PATH = "js/js.translator/testData/"
     }
 
-}
+}

js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt

@@ -112,6 +112,9 @@ class TestModuleCache(val files: MutableMap<String, FileCache>) {
                 files.remove(path)
             }
 
+            override fun invalidate() {
+            }
+
             override fun commitLibraryInfo(libraryPath: String, moduleName: String, flatHash: ULong, transHash: ULong, configHash: ULong) {
                 storedModuleName = moduleName
             }

js/js.tests/tests-gen/org/jetbrains/kotlin/incremental/InvalidationTestGenerated.java

@@ -100,6 +100,11 @@ public class InvalidationTestGenerated extends AbstractInvalidationTest {
         runTest("js/js.translator/testData/incremental/invalidation/jsExport/");
     }
 
+    @TestMetadata("mainModuleInvalidation")
+    public void testMainModuleInvalidation() throws Exception {
+        runTest("js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/");
+    }
+
     @TestMetadata("moveFilesBetweenModules")
     public void testMoveFilesBetweenModules() throws Exception {
         runTest("js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/");

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/l1.kt

@@ -0,0 +1 @@
+fun qux() = 42

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/module.info

@@ -0,0 +1,6 @@
+STEP 0:
+    dependencies: stdlib
+    dirty: l1.kt
+STEP 1:
+    dependencies: stdlib
+    flags: FP

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt

@@ -0,0 +1,2 @@
+
+fun box() = qux()

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt.1.txt

@@ -0,0 +1,2 @@
+
+fun box() = qux() + 88

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m0.kt

@@ -0,0 +1,2 @@
+
+fun box0() = qux()

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m1.kt

@@ -0,0 +1,2 @@
+
+fun box1() = qux()

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/module.info

@@ -0,0 +1,9 @@
+STEP 0:
+    dependencies: stdlib, lib1
+    dirty: m.kt, m0.kt, m1.kt
+
+STEP 1:
+    dependencies: stdlib, lib1
+    modifications:
+        U : m.kt.1.txt -> m.kt
+    dirty: m.kt

js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/project.info

@@ -0,0 +1,7 @@
+MODULES: lib1, main
+
+STEP 0:
+    libs: lib1, main
+
+STEP 1:
+    libs: lib1, main

js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info

@@ -5,3 +5,6 @@ STEP 1:
     dependencies: stdlib
     modifications:
         D : l1_1.kt
+STEP 2:
+    dependencies: stdlib
+    flags: FP

js/js.translator/testData/incremental/invalidation/removeUnusedFile/main/module.info

@@ -1,7 +1,8 @@
 STEP 0:
     dependencies: stdlib, lib1
     dirty: m.kt
-
 STEP 1:
     dependencies: stdlib, lib1
-    flags: FP
+STEP 2:
+    dependencies: stdlib, lib1
+    flags: FP

js/js.translator/testData/incremental/invalidation/removeUnusedFile/project.info

@@ -4,4 +4,7 @@ STEP 0:
     libs: lib1, main
 
 STEP 1:
-    libs: lib1, main
+    libs: lib1, main
+
+STEP 2:
+    libs: lib1, main

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt

@@ -26,9 +26,12 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT
 import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.PRODUCTION
 import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile
 import org.jetbrains.kotlin.gradle.utils.getValue
+import org.jetbrains.kotlin.gradle.utils.toHexString
 import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics
 import org.jetbrains.kotlin.statistics.metrics.StringMetrics
 import java.io.File
+import java.nio.charset.StandardCharsets
+import java.security.MessageDigest
 import javax.inject.Inject
 
 @CacheableTask
@@ -65,10 +68,6 @@ abstract class KotlinJsIrLink @Inject constructor(
     @get:Internal
     internal val propertiesProvider = PropertiesProvider(project)
 
-    @get:Inject
-    open val fileHasher: FileHasher
-        get() = throw UnsupportedOperationException()
-
     @get:Input
     internal val incrementalJsIr: Boolean = propertiesProvider.incrementalJsIr
 
@@ -125,15 +124,17 @@ abstract class KotlinJsIrLink @Inject constructor(
             )
         }
         if (incrementalJsIr && mode == DEVELOPMENT) {
+            val digest = MessageDigest.getInstance("SHA-256")
             args.cacheDirectories = args.libraries?.splitByPathSeparator()
                 ?.map {
                     val file = File(it)
+                    val hash = digest.digest(file.normalize().absolutePath.toByteArray(StandardCharsets.UTF_8)).toHexString()
                     rootCacheDirectory
                         .resolve(file.nameWithoutExtension)
+                        .resolve(hash)
                         .also {
                             it.mkdirs()
                         }
-                        .resolve(fileHasher.hash(file).toString())
                 }
                 ?.plus(rootCacheDirectory.resolve(entryModule.get().asFile.name))
                 ?.let {

libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt

@@ -43,7 +43,7 @@ fun getCacheDirectory(
     return File(cacheDirectory, computeDependenciesHash(dependency))
 }
 
-private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') }
+internal fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') }
 
 private fun computeDependenciesHash(dependency: ResolvedDependency): String {
     val allArtifactsPaths =
@@ -103,4 +103,4 @@ internal class GradleLoggerAdapter(private val gradleLogger: Logger) : org.jetbr
     override fun warning(message: String) = gradleLogger.warn(message)
     override fun error(message: String) = kotlin.error(message)
     override fun fatal(message: String): Nothing = kotlin.error(message)
-}
+}