[JS IR] Rework incremental cache invalidation

Replace loading the whole world IR with loading only exported (reachable) IR.

 For that purpose the direct and inverse dependency graph is used.
 It is stored in a cache directory and the cache updater keeps it up to date.
 If after loading it is found that other files must be also implicitly rebuilt
 (see rebuilt reasons below), IR for that files also will be loaded.
 This algorithm repeats until the number of implicitly rebuilt files is not 0.

 More rebuilt reasons (dirty state) have been added:
  - added file: this is a new file;
  - modified ir: ir of the file has been updated;
  - updated exports: exports from the file have been added or removed
    (e.g. a function has been used from another file);
  - updated inline imports: imported inline function has been modified
    (transitively);
  - removed inverse depends: a dependent file has been removed;
  - removed direct depends: a dependency file has been removed;
  - removed file: this file has been removed.

 Incremental cache tests has been refactored:
  - The supporting of all rebuilt reasons (dirty states) has been added;
  - New file name format "*.$suffix.kt" for the test steps has been allowed,
    so the syntax highlight works now;
  - Explicit stdlib dependency usage has been removed.

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -213,43 +213,42 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             messageCollector.report(INFO, arguments.cacheDirectories ?: "")
             messageCollector.report(INFO, libraries.toString())
 
-            val includes = arguments.includes!!
-
-            var start = System.currentTimeMillis()
+            val start = System.currentTimeMillis()
 
             val cacheUpdater = CacheUpdater(
-                includes,
-                libraries,
-                configurationJs,
-                cacheDirectories,
-                { IrFactoryImplForJsIC(WholeWorldStageController()) },
-                mainCallArguments,
-                ::buildCacheForModuleFiles
+                mainModule = arguments.includes!!,
+                allModules = libraries,
+                icCachePaths = cacheDirectories,
+                compilerConfiguration = configurationJs,
+                irFactory = { IrFactoryImplForJsIC(WholeWorldStageController()) },
+                mainArguments = mainCallArguments,
+                executor = ::buildCacheForModuleFiles
             )
-            cacheUpdater.actualizeCaches { updateStatus, updatedModule ->
+
+            var tp = System.currentTimeMillis()
+            messageCollector.report(INFO, "IC cache updater initialization: ${tp - start}ms")
+
+            val artifacts = cacheUpdater.actualizeCaches {
                 val now = System.currentTimeMillis()
-                fun reportCacheStatus(status: String, removed: Set<String> = emptySet(), updated: Set<String> = emptySet()) {
-                    messageCollector.report(INFO, "IC per-file is $status duration ${now - start}ms; module [${File(updatedModule).name}]")
-                    removed.forEach { messageCollector.report(INFO, "  Removed: $it") }
-                    updated.forEach { messageCollector.report(INFO, "  Updated: $it") }
-                }
-                when (updateStatus) {
-                    is CacheUpdateStatus.FastPath -> reportCacheStatus("up-to-date; fast check")
-                    is CacheUpdateStatus.NoDirtyFiles -> reportCacheStatus("up-to-date; full check", updateStatus.removed)
-                    is CacheUpdateStatus.Dirty -> {
-                        var updated = updateStatus.updated
-                        val status = StringBuilder("dirty").apply {
-                            if (updateStatus.updatedAll) {
-                                append("; all ${updated.size} sources updated")
-                                updated = emptySet()
-                            }
-                            append("; cache building")
-                        }.toString()
-                        reportCacheStatus(status, updateStatus.removed, updated)
+                messageCollector.report(INFO, "IC $it: ${now - tp}ms")
+                tp = now
+            }
+
+            messageCollector.report(INFO, "IC rebuilt overall time: ${System.currentTimeMillis() - start}ms")
+
+            for ((libFile, srcFiles) in cacheUpdater.getDirtyFileStats()) {
+                val isCleanBuild = srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) }
+                val msg = if (isCleanBuild) "fully rebuilt" else "partially rebuilt"
+                messageCollector.report(INFO, "module [${File(libFile.path).name}] was $msg")
+                if (!isCleanBuild) {
+                    for ((srcFile, stat) in srcFiles) {
+                        val statStr = stat.joinToString { it.str }
+                        messageCollector.report(INFO, "  file [${File(srcFile.path).name}]: ($statStr)")
                     }
                 }
-                start = now
             }
+
+            artifacts
         } else emptyList()
 
         // Run analysis if main module is sources

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/JsIrBackendContext.kt

@@ -50,6 +50,12 @@ import org.jetbrains.kotlin.types.isNullable
 import org.jetbrains.kotlin.util.collectionUtils.filterIsInstanceMapNotNull
 import org.jetbrains.kotlin.utils.addToStdlib.cast
 
+enum class IcCompatibleIr2Js(val isCompatible: Boolean, val incrementalCacheEnabled: Boolean) {
+    DISABLED(false, false),
+    COMPATIBLE(true, false),
+    IC_MODE(true, true)
+}
+
 class JsIrBackendContext(
     val module: ModuleDescriptor,
     override val irBuiltIns: IrBuiltIns,
@@ -65,7 +71,7 @@ class JsIrBackendContext(
     val safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null,
     override val mapping: JsMapping = JsMapping(),
     val granularity: JsGenerationGranularity = JsGenerationGranularity.WHOLE_PROGRAM,
-    val icCompatibleIr2Js: Boolean = false,
+    val icCompatibleIr2Js: IcCompatibleIr2Js = IcCompatibleIr2Js.DISABLED,
 ) : JsCommonBackendContext {
     val polyfills = JsPolyfills()
     val fieldToInitializer: MutableMap<IrField, IrExpression> = mutableMapOf()

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt

@@ -128,7 +128,7 @@ fun compileIr(
         safeExternalBoolean = safeExternalBoolean,
         safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic,
         granularity = granularity,
-        icCompatibleIr2Js = icCompatibleIr2Js,
+        icCompatibleIr2Js = if (icCompatibleIr2Js) IcCompatibleIr2Js.COMPATIBLE else IcCompatibleIr2Js.DISABLED
     )
 
     // Load declarations referenced during `context` initialization
@@ -166,4 +166,4 @@ fun generateJsCode(
 
     val transformer = IrModuleToJsTransformer(context, null, true, nameTables)
     return transformer.generateModule(listOf(moduleFragment)).outputs[TranslationMode.FULL]!!.jsCode
-}
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compilerWithIC.kt

@@ -9,29 +9,27 @@ import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig
 import org.jetbrains.kotlin.backend.common.phaser.PhaserState
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
-import org.jetbrains.kotlin.ir.backend.js.ic.ArtifactCache
-import org.jetbrains.kotlin.ir.backend.js.ic.JsMultiModuleCache
-import org.jetbrains.kotlin.ir.backend.js.ic.ModuleArtifact
 import org.jetbrains.kotlin.ir.backend.js.lower.collectNativeImplementations
 import org.jetbrains.kotlin.ir.backend.js.lower.generateJsTests
 import org.jetbrains.kotlin.ir.backend.js.lower.moveBodilessDeclarationsToSeparatePlace
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.*
+import org.jetbrains.kotlin.ir.declarations.IrFile
 import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
 import org.jetbrains.kotlin.ir.util.ExternalDependenciesGenerator
 import org.jetbrains.kotlin.ir.util.noUnboundLeft
 import org.jetbrains.kotlin.js.config.RuntimeDiagnostic
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.psi2ir.descriptors.IrBuiltInsOverDescriptors
-import org.jetbrains.kotlin.serialization.js.ModuleKind
 
 @Suppress("UNUSED_PARAMETER")
 @OptIn(ObsoleteDescriptorBasedAPI::class)
 fun compileWithIC(
-    module: IrModuleFragment,
+    mainModule: IrModuleFragment,
     configuration: CompilerConfiguration,
     deserializer: JsIrLinker,
-    dependencies: Collection<IrModuleFragment>,
+    allModules: Collection<IrModuleFragment>,
+    filesToLower: Collection<IrFile>,
     mainArguments: List<String>? = null,
     exportedDeclarations: Set<FqName> = emptySet(),
     generateFullJs: Boolean = true,
@@ -45,22 +43,16 @@ fun compileWithIC(
     baseClassIntoMetadata: Boolean = false,
     lowerPerModule: Boolean = false,
     safeExternalBoolean: Boolean = false,
-    safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null,
-    filesToLower: Set<String>?,
-    artifactCache: ArtifactCache,
-) {
-
-    val mainModule = module
-    val allModules = dependencies
-    val moduleDescriptor = module.descriptor
-    val irBuiltIns = module.irBuiltins
+    safeExternalBooleanDiagnostic: RuntimeDiagnostic? = null
+): List<JsIrFragmentAndBinaryAst> {
+    val irBuiltIns = mainModule.irBuiltins
     val symbolTable = (irBuiltIns as IrBuiltInsOverDescriptors).symbolTable
 
     val context = JsIrBackendContext(
-        moduleDescriptor,
+        mainModule.descriptor,
         irBuiltIns,
         symbolTable,
-        module,
+        mainModule,
         exportedDeclarations,
         configuration,
         es6mode = es6mode,
@@ -68,7 +60,7 @@ fun compileWithIC(
         baseClassIntoMetadata = baseClassIntoMetadata,
         safeExternalBoolean = safeExternalBoolean,
         safeExternalBooleanDiagnostic = safeExternalBooleanDiagnostic,
-        icCompatibleIr2Js = true,
+        icCompatibleIr2Js = IcCompatibleIr2Js.IC_MODE
     )
 
     // Load declarations referenced during `context` initialization
@@ -79,7 +71,7 @@ fun compileWithIC(
     symbolTable.noUnboundLeft("Unbound symbols at the end of linker")
 
     allModules.forEach {
-        collectNativeImplementations(context, module)
+        collectNativeImplementations(context, it)
         moveBodilessDeclarationsToSeparatePlace(context, it)
     }
 
@@ -93,18 +85,15 @@ fun compileWithIC(
         relativeRequirePath = relativeRequirePath,
     )
 
-    val dirtyFiles = filesToLower?.let { dirties ->
-        module.files.filter { it.fileEntry.name in dirties }
-    } ?: module.files
-
-    val astAndFragments = transformer.generateBinaryAst(dirtyFiles, allModules)
-    astAndFragments.forEach {
-        artifactCache.saveFragment(it.srcPath, it.fragment)
-        artifactCache.saveBinaryAst(it.srcPath, it.binaryAst)
-    }
+    return transformer.generateBinaryAst(filesToLower, allModules)
 }
 
-fun lowerPreservingTags(modules: Iterable<IrModuleFragment>, context: JsIrBackendContext, phaseConfig: PhaseConfig, controller: WholeWorldStageController) {
+fun lowerPreservingTags(
+    modules: Iterable<IrModuleFragment>,
+    context: JsIrBackendContext,
+    phaseConfig: PhaseConfig,
+    controller: WholeWorldStageController
+) {
     // Lower all the things
     controller.currentStage = 0
 

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -0,0 +1,579 @@
+/* * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import org.jetbrains.kotlin.config.CompilerConfiguration
+import org.jetbrains.kotlin.ir.backend.js.*
+import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrFragmentAndBinaryAst
+import org.jetbrains.kotlin.ir.declarations.IrFactory
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.util.*
+import org.jetbrains.kotlin.js.config.JSConfigurationKeys
+import org.jetbrains.kotlin.konan.properties.propertyList
+import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.name.FqName
+import java.io.File
+import java.util.EnumSet
+
+
+fun interface CacheExecutor {
+    fun execute(
+        mainModule: IrModuleFragment,
+        allModules: Collection<IrModuleFragment>,
+        deserializer: JsIrLinker,
+        configuration: CompilerConfiguration,
+        dirtyFiles: Collection<IrFile>,
+        exportedDeclarations: Set<FqName>,
+        mainArguments: List<String>?
+    ): List<JsIrFragmentAndBinaryAst>
+}
+
+enum class DirtyFileState(val str: String) {
+    ADDED_FILE("added file"),
+    MODIFIED_IR("modified ir"),
+    UPDATED_EXPORTS("updated exports"),
+    UPDATED_INLINE_IMPORTS("updated inline imports"),
+    REMOVED_INVERSE_DEPENDS("removed inverse depends"),
+    REMOVED_DIRECT_DEPENDS("removed direct depends"),
+    REMOVED_FILE("removed file")
+}
+
+class CacheUpdater(
+    mainModule: String,
+    allModules: Collection<String>,
+    icCachePaths: Collection<String>,
+    private val compilerConfiguration: CompilerConfiguration,
+    private val irFactory: () -> IrFactory,
+    private val mainArguments: List<String>?,
+    private val executor: CacheExecutor
+) {
+    private val hashCalculator = InlineFunctionTransitiveHashCalculator()
+    private val transitiveHashes
+        get() = hashCalculator.transitiveHashes
+
+    private val libraries = loadLibraries(allModules)
+    private val dependencyGraph = buildDependenciesGraph(libraries)
+    private val configHash = compilerConfiguration.configHashForIC()
+
+    private val cacheMap = libraries.values.zip(icCachePaths).toMap()
+
+    private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)
+    private val mainLibrary = libraries[mainLibraryFile] ?: notFoundIcError("main library", mainLibraryFile)
+
+    private val incrementalCaches = libraries.entries.associate { (libFile, lib) ->
+        val cachePath = cacheMap[lib] ?: notFoundIcError("cache path", KotlinLibraryFile(lib))
+        libFile to IncrementalCache(lib, cachePath)
+    }
+
+    private val dirtyFileStats = KotlinSourceFileMutableMap<EnumSet<DirtyFileState>>()
+
+    fun getDirtyFileStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats
+
+    private fun MutableMap<KotlinSourceFile, EnumSet<DirtyFileState>>.addDirtFileStat(srcFile: KotlinSourceFile, state: DirtyFileState) {
+        when (val stats = this[srcFile]) {
+            null -> this[srcFile] = EnumSet.of(state)
+            else -> stats.add(state)
+        }
+    }
+
+    private fun getLibIncrementalCache(libFile: KotlinLibraryFile) =
+        incrementalCaches[libFile] ?: notFoundIcError("incremental cache", libFile)
+
+    private fun loadLibraries(allModules: Collection<String>): Map<KotlinLibraryFile, KotlinLibrary> {
+        val allResolvedDependencies = jsResolveLibraries(
+            allModules,
+            compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList(),
+            compilerConfiguration[IrMessageLogger.IR_MESSAGE_LOGGER].toResolverLogger()
+        )
+
+        return allResolvedDependencies.getFullList().associateBy { KotlinLibraryFile(it) }
+    }
+
+    private fun buildDependenciesGraph(libraries: Map<KotlinLibraryFile, KotlinLibrary>): Map<KotlinLibrary, List<KotlinLibrary>> {
+        val nameToKotlinLibrary = libraries.values.associateBy { it.moduleName }
+        return libraries.values.associateWith {
+            it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
+                nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName")
+            }
+        }
+    }
+
+    private class DirtyFileExports(
+        override val inverseDependencies: KotlinSourceFileMutableMap<Set<IdSignature>> = KotlinSourceFileMutableMap()
+    ) : KotlinSourceFileExports() {
+        override fun getExportedSignatures(): Set<IdSignature> = allExportedSignatures
+
+        val allExportedSignatures = mutableSetOf<IdSignature>()
+    }
+
+    private class DirtyFileMetadata(
+        val maybeImportedSignatures: Collection<IdSignature>,
+
+        val oldDirectDependencies: KotlinSourceFileMap<*>,
+
+        override val inverseDependencies: KotlinSourceFileMutableMap<MutableSet<IdSignature>> = KotlinSourceFileMutableMap(),
+        override val directDependencies: KotlinSourceFileMutableMap<MutableSet<IdSignature>> = KotlinSourceFileMutableMap(),
+
+        override val importedInlineFunctions: MutableMap<IdSignature, ICHash> = mutableMapOf()
+    ) : KotlinSourceFileMetadata() {
+        fun addInverseDependency(lib: KotlinLibraryFile, src: KotlinSourceFile, signature: IdSignature) =
+            inverseDependencies.addSignature(lib, src, signature)
+
+        fun addDirectDependency(lib: KotlinLibraryFile, src: KotlinSourceFile, signature: IdSignature) =
+            directDependencies.addSignature(lib, src, signature)
+    }
+
+    private class UpdatedDependenciesMetadata(oldMetadata: KotlinSourceFileMetadata) : KotlinSourceFileMetadata() {
+        private val oldInverseDependencies = oldMetadata.inverseDependencies
+        private val newExportedSignatures: Set<IdSignature> by lazy { inverseDependencies.flatSignatures() }
+
+        var importedInlineFunctionsModified = false
+
+        override val inverseDependencies = oldMetadata.inverseDependencies.toMutable()
+        override val directDependencies = oldMetadata.directDependencies.toMutable()
+        override val importedInlineFunctions = oldMetadata.importedInlineFunctions
+
+        override fun getExportedSignatures(): Set<IdSignature> = newExportedSignatures
+
+        fun isExportedSignaturesUpdated() = newExportedSignatures != oldInverseDependencies.flatSignatures()
+    }
+
+    private fun addFilesWithRemovedDependencies(
+        modifiedFiles: KotlinSourceFileMap<KotlinSourceFileMetadata>, removedFiles: KotlinSourceFileMap<KotlinSourceFileMetadata>
+    ): KotlinSourceFileMap<KotlinSourceFileMetadata> {
+        val extraModifiedLibFiles = KotlinSourceFileMutableMap<KotlinSourceFileMetadata>()
+
+        fun addDependenciesToExtraModifiedFiles(dependencies: KotlinSourceFileMap<*>, dirtyState: DirtyFileState) {
+            for ((dependentLib, dependentFiles) in dependencies) {
+                val dependentCache = incrementalCaches[dependentLib] ?: continue
+                val alreadyModifiedFiles = modifiedFiles[dependentLib] ?: emptyMap()
+                val alreadyRemovedFiles = removedFiles[dependentLib] ?: emptyMap()
+                val extraModifiedFiles by lazy(LazyThreadSafetyMode.NONE) { extraModifiedLibFiles.getOrPutFiles(dependentLib) }
+                val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(dependentLib) }
+                for (dependentFile in dependentFiles.keys) {
+                    when (dependentFile) {
+                        in alreadyModifiedFiles -> continue
+                        in alreadyRemovedFiles -> continue
+                        in extraModifiedFiles -> continue
+                        else -> {
+                            val dependentMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile)
+                            extraModifiedFiles[dependentFile] = dependentMetadata
+                            fileStats.addDirtFileStat(dependentFile, dirtyState)
+                        }
+                    }
+                }
+            }
+        }
+
+        removedFiles.forEachFile { _, _, removedFileMetadata ->
+            addDependenciesToExtraModifiedFiles(removedFileMetadata.directDependencies, DirtyFileState.REMOVED_INVERSE_DEPENDS)
+            addDependenciesToExtraModifiedFiles(removedFileMetadata.inverseDependencies, DirtyFileState.REMOVED_DIRECT_DEPENDS)
+        }
+
+        if (extraModifiedLibFiles.isNotEmpty()) {
+            extraModifiedLibFiles.copyFilesFrom(modifiedFiles)
+            return extraModifiedLibFiles
+        }
+        return modifiedFiles
+    }
+
+    private fun loadModifiedFiles(): KotlinSourceFileMap<KotlinSourceFileMetadata> {
+        val removedFilesMetadata = mutableMapOf<KotlinLibraryFile, Map<KotlinSourceFile, KotlinSourceFileMetadata>>()
+
+        val modifiedFiles = KotlinSourceFileMap(incrementalCaches.entries.associate { (lib, cache) ->
+            val (dirtyFiles, removedFiles, newFiles) = cache.collectModifiedFiles(configHash)
+
+            val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(lib) }
+            newFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) }
+            removedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.REMOVED_FILE) }
+            dirtyFiles.forEach {
+                if (it.key !in newFiles) {
+                    fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR)
+                }
+            }
+
+            if (removedFiles.isNotEmpty()) {
+                removedFilesMetadata[lib] = removedFiles
+            }
+
+            lib to dirtyFiles
+        })
+
+        return addFilesWithRemovedDependencies(modifiedFiles, KotlinSourceFileMap(removedFilesMetadata))
+    }
+
+    private fun collectExportedSymbolsForDirtyFiles(
+        dirtyFiles: KotlinSourceFileMap<KotlinSourceFileMetadata>
+    ): KotlinSourceFileMutableMap<KotlinSourceFileExports> {
+        val exportedSymbols = KotlinSourceFileMutableMap<KotlinSourceFileExports>()
+
+        for ((libFile, srcFiles) in dirtyFiles) {
+            val exportedSymbolFiles = mutableMapOf<KotlinSourceFile, KotlinSourceFileExports>()
+            for ((srcFile, srcFileMetadata) in srcFiles) {
+                val loadingFileExports = DirtyFileExports()
+                for ((dependentLib, dependentFiles) in srcFileMetadata.inverseDependencies) {
+                    val dependentCache = incrementalCaches[dependentLib] ?: continue
+                    val dirtyLibFiles = dirtyFiles[dependentLib] ?: emptyMap()
+                    for (dependentFile in dependentFiles.keys) {
+                        if (dependentFile !in dirtyLibFiles) {
+                            val dependentSrcFileMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile)
+                            dependentSrcFileMetadata.directDependencies[libFile, srcFile]?.let {
+                                loadingFileExports.inverseDependencies[dependentLib, dependentFile] = it
+                                loadingFileExports.allExportedSignatures += it
+                            }
+                        }
+                    }
+                }
+                exportedSymbolFiles[srcFile] = loadingFileExports
+            }
+            if (exportedSymbolFiles.isNotEmpty()) {
+                exportedSymbols[libFile] = exportedSymbolFiles
+            }
+        }
+        return exportedSymbols
+    }
+
+    private fun rebuildDirtySourceMetadata(
+        jsIrLinker: JsIrLinker,
+        loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
+        dirtySrcFiles: KotlinSourceFileMap<KotlinSourceFileExports>,
+    ): KotlinSourceFileMap<DirtyFileMetadata> {
+        val idSignatureToFile = mutableMapOf<IdSignature, Pair<KotlinLibraryFile, KotlinSourceFile>>()
+        val updatedMetadata = KotlinSourceFileMutableMap<DirtyFileMetadata>()
+
+        for ((lib, irModule) in loadedFragments) {
+            val moduleDeserializer = jsIrLinker.moduleDeserializer(irModule.descriptor)
+            val incrementalCache = getLibIncrementalCache(lib)
+            for (fileDeserializer in moduleDeserializer.fileDeserializers()) {
+                val libSrcFile = KotlinSourceFile(fileDeserializer.file)
+
+                val reachableSignatures = fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
+                val allImplementedSignatures = fileDeserializer.symbolDeserializer.deserializedSymbols.keys
+                val maybeImportedSignatures = reachableSignatures.keys.toMutableSet()
+                for (signature in allImplementedSignatures) {
+                    if (signature in reachableSignatures) {
+                        idSignatureToFile[signature] = lib to libSrcFile
+                        maybeImportedSignatures.remove(signature)
+                    }
+                }
+
+                val metadata = incrementalCache.fetchSourceFileFullMetadata(libSrcFile)
+                updatedMetadata[lib, libSrcFile] = DirtyFileMetadata(maybeImportedSignatures, metadata.directDependencies)
+            }
+        }
+
+        for ((libFile, srcFiles) in updatedMetadata) {
+            val libDirtySrcFiles = dirtySrcFiles[libFile] ?: continue
+            for ((srcFile, internalHeader) in srcFiles) {
+                val dirtySrcFile = libDirtySrcFiles[srcFile] ?: continue
+                dirtySrcFile.inverseDependencies.forEachFile { dependentLibFile, dependentSrcFile, signatures ->
+                    signatures.forEach {
+                        val (dependencyLib, dependencyFile) = idSignatureToFile[it] ?: (libFile to srcFile)
+                        updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata ->
+                            dependencyMetadata.addInverseDependency(dependentLibFile, dependentSrcFile, it)
+                        } ?: notFoundIcError("metadata", dependencyLib, dependencyFile)
+                    }
+                }
+
+                for (importedSignature in internalHeader.maybeImportedSignatures) {
+                    val (dependencyLib, dependencyFile) = idSignatureToFile[importedSignature] ?: continue
+                    internalHeader.addDirectDependency(dependencyLib, dependencyFile, importedSignature)
+                    transitiveHashes[importedSignature]?.let { internalHeader.importedInlineFunctions[importedSignature] = it }
+
+                    updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata ->
+                        dependencyMetadata.addInverseDependency(libFile, srcFile, importedSignature)
+                    } ?: notFoundIcError("metadata", dependencyLib, dependencyFile)
+                }
+            }
+        }
+
+        val result = KotlinSourceFileMutableMap<DirtyFileMetadata>()
+
+        for ((libFile, sourceFiles) in dirtySrcFiles) {
+            val incrementalCache = getLibIncrementalCache(libFile)
+            val srcFileUpdatedMetadata = updatedMetadata[libFile] ?: notFoundIcError("metadata", libFile)
+            for (srcFile in sourceFiles.keys) {
+                val srcMetadata = srcFileUpdatedMetadata[srcFile] ?: notFoundIcError("metadata", libFile, srcFile)
+                incrementalCache.updateSourceFileMetadata(srcFile, srcMetadata)
+                result[libFile, srcFile] = srcMetadata
+            }
+        }
+
+        return result
+    }
+
+    private fun KotlinSourceFileMutableMap<UpdatedDependenciesMetadata>.addNewMetadata(
+        libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, oldMetadata: KotlinSourceFileMetadata
+    ) = this[libFile, srcFile] ?: UpdatedDependenciesMetadata(oldMetadata).also {
+        this[libFile, srcFile] = it
+    }
+
+    private fun KotlinSourceFileMutableMap<UpdatedDependenciesMetadata>.addDependenciesWithUpdatedSignatures(
+        libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata
+    ) {
+        // go through dependencies and collect dependencies with updated signatures
+        for ((dependencyLibFile, dependencySrcFiles) in srcFileMetadata.directDependencies) {
+            val dependencyCache = getLibIncrementalCache(dependencyLibFile)
+            for ((dependencySrcFile, newSignatures) in dependencySrcFiles) {
+                val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(dependencySrcFile)
+                val oldSignatures = dependencySrcMetadata.inverseDependencies[libFile, srcFile] ?: emptySet()
+                if (oldSignatures == newSignatures) {
+                    continue
+                }
+                val newMetadata = addNewMetadata(dependencyLibFile, dependencySrcFile, dependencySrcMetadata)
+                newMetadata.inverseDependencies[libFile, srcFile] = newSignatures
+            }
+        }
+    }
+
+    private fun KotlinSourceFileMutableMap<UpdatedDependenciesMetadata>.addDependenciesWithRemovedInverseDependencies(
+        libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata
+    ) {
+        // go through old dependencies and look for removed dependencies
+        for ((oldDependencyLibFile, oldDependencySrcFiles) in srcFileMetadata.oldDirectDependencies) {
+            val dependencyCache = incrementalCaches[oldDependencyLibFile] ?: continue
+            val newDirectDependencyFiles = srcFileMetadata.directDependencies[oldDependencyLibFile] ?: emptyMap()
+            for (oldDependencySrcFile in oldDependencySrcFiles.keys) {
+                if (oldDependencySrcFile in newDirectDependencyFiles) {
+                    continue
+                }
+                val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(oldDependencySrcFile)
+                if (dependencySrcMetadata.inverseDependencies[libFile, srcFile] != null) {
+                    val newMetadata = addNewMetadata(oldDependencyLibFile, oldDependencySrcFile, dependencySrcMetadata)
+                    newMetadata.inverseDependencies.removeFile(libFile, srcFile)
+                }
+            }
+        }
+    }
+
+    private fun KotlinSourceFileMutableMap<UpdatedDependenciesMetadata>.addDependentsWithUpdatedImports(
+        libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata
+    ) {
+        // go through dependent files and check if their inline imports were modified
+        for ((dependentLibFile, dependentSrcFiles) in srcFileMetadata.inverseDependencies) {
+            val dependentCache = incrementalCaches[dependentLibFile] ?: continue
+            for ((dependentSrcFile, newSignatures) in dependentSrcFiles) {
+                val dependentSrcMetadata = dependentCache.fetchSourceFileFullMetadata(dependentSrcFile)
+                val dependentSignatures = dependentSrcMetadata.directDependencies[libFile, srcFile] ?: emptySet()
+                val importedInlineModified = dependentSrcMetadata.importedInlineFunctions.any {
+                    transitiveHashes[it.key]?.let { newHash -> newHash != it.value } ?: (it.key in dependentSignatures)
+                }
+                if (importedInlineModified) {
+                    val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata)
+                    newMetadata.importedInlineFunctionsModified = true
+                } else if (dependentSignatures != newSignatures) {
+                    val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata)
+                    newMetadata.directDependencies[libFile, srcFile] = newSignatures
+                }
+            }
+        }
+    }
+
+    private fun collectFilesWithModifiedExportsOrInlineImports(
+        loadedDirtyFiles: KotlinSourceFileMap<DirtyFileMetadata>
+    ): KotlinSourceFileMap<UpdatedDependenciesMetadata> {
+        val filesWithModifiedExports = KotlinSourceFileMutableMap<UpdatedDependenciesMetadata>()
+
+        loadedDirtyFiles.forEachFile { libFile, srcFile, srcFileMetadata ->
+            filesWithModifiedExports.addDependenciesWithUpdatedSignatures(libFile, srcFile, srcFileMetadata)
+            filesWithModifiedExports.addDependenciesWithRemovedInverseDependencies(libFile, srcFile, srcFileMetadata)
+            filesWithModifiedExports.addDependentsWithUpdatedImports(libFile, srcFile, srcFileMetadata)
+        }
+
+        return filesWithModifiedExports
+    }
+
+    private fun collectFilesToRebuildSignatures(
+        filesWithModifiedExports: KotlinSourceFileMap<UpdatedDependenciesMetadata>
+    ): KotlinSourceFileMap<KotlinSourceFileExports> {
+        val libFilesToRebuild = KotlinSourceFileMutableMap<KotlinSourceFileExports>()
+
+        for ((libFile, srcFiles) in filesWithModifiedExports) {
+            val filesToRebuild by lazy(LazyThreadSafetyMode.NONE) { libFilesToRebuild.getOrPutFiles(libFile) }
+            val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(libFile) }
+            val cache = getLibIncrementalCache(libFile)
+
+            for ((srcFile, srcFileMetadata) in srcFiles) {
+                val isSignatureUpdated = srcFileMetadata.isExportedSignaturesUpdated()
+                if (isSignatureUpdated || srcFileMetadata.importedInlineFunctionsModified) {
+                    // if exported signatures or imported inline functions were modified - rebuild
+                    filesToRebuild[srcFile] = srcFileMetadata
+                    if (isSignatureUpdated) {
+                        fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_EXPORTS)
+                    }
+                    if (srcFileMetadata.importedInlineFunctionsModified) {
+                        fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_INLINE_IMPORTS)
+                    }
+                } else {
+                    // if signatures and inline functions are the same - just update cache metadata
+                    cache.updateSourceFileMetadata(srcFile, srcFileMetadata)
+                }
+            }
+        }
+
+        return libFilesToRebuild
+    }
+
+    private fun buildModuleArtifactsAndCommitCache(
+        jsIrLinker: JsIrLinker,
+        loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
+        rebuiltFileFragments: List<JsIrFragmentAndBinaryAst>
+    ): List<ModuleArtifact> {
+        val fragmentToLibName = loadedFragments.entries.associate { it.value to it.key }
+
+        val rebuiltSrcFiles = rebuiltFileFragments.groupBy {
+            fragmentToLibName[it.irFile.module] ?: notFoundIcError("loaded fragment lib name", srcFile = KotlinSourceFile(it.irFile))
+        }
+
+        val visited = mutableSetOf<KotlinLibrary>()
+        val artifacts = mutableListOf<ModuleArtifact>()
+        fun addArtifact(lib: KotlinLibrary) {
+            if (visited.add(lib)) {
+                dependencyGraph[lib]?.forEach(::addArtifact)
+
+                val libFile = KotlinLibraryFile(lib)
+                val incrementalCache = getLibIncrementalCache(libFile)
+                val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
+                val libRebuiltFiles = rebuiltSrcFiles[libFile]?.associateBy { KotlinSourceFile(it.irFile) } ?: emptyMap()
+                val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor)
+
+                val signatureToIndexMapping = moduleDeserializer.fileDeserializers().associate {
+                    KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
+                }
+                artifacts += incrementalCache.buildModuleArtifactAndCommitCache(
+                    moduleName = libFragment.name.asString(),
+                    rebuiltFileFragments = libRebuiltFiles,
+                    signatureToIndexMapping = signatureToIndexMapping
+                )
+            }
+        }
+
+        addArtifact(mainLibrary)
+
+        return artifacts
+    }
+
+    fun actualizeCaches(eventCallback: (String) -> Unit = {}): List<ModuleArtifact> {
+        dirtyFileStats.clear()
+
+        val modifiedFiles = loadModifiedFiles()
+        val dirtyFileExports = collectExportedSymbolsForDirtyFiles(modifiedFiles)
+
+        val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, mainLibrary, dependencyGraph, irFactory())
+        var loadedIr = jsIrLinkerLoader.loadIr(dirtyFileExports)
+
+        eventCallback("initial loading of updated files")
+
+        var iterations = 0
+        var lastDirtyFiles: KotlinSourceFileMap<KotlinSourceFileExports> = dirtyFileExports
+
+        while (true) {
+            hashCalculator.updateTransitiveHashes(loadedIr.loadedFragments.values)
+
+            val dirtyHeaders = rebuildDirtySourceMetadata(loadedIr.linker, loadedIr.loadedFragments, lastDirtyFiles)
+
+            val filesWithModifiedExportsOrImports = collectFilesWithModifiedExportsOrInlineImports(dirtyHeaders)
+
+            val filesToRebuild = collectFilesToRebuildSignatures(filesWithModifiedExportsOrImports)
+
+            eventCallback("actualization iteration $iterations")
+            if (filesToRebuild.isEmpty()) {
+                break
+            }
+
+            loadedIr = jsIrLinkerLoader.loadIr(filesToRebuild)
+            iterations++
+
+            lastDirtyFiles = filesToRebuild
+            dirtyFileExports.copyFilesFrom(filesToRebuild)
+        }
+
+        if (iterations != 0) {
+            loadedIr = jsIrLinkerLoader.loadIr(dirtyFileExports)
+            eventCallback("final loading of updated files")
+        }
+
+        val rebuiltFragments = executor.execute(
+            mainModule = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main lib loaded fragment", mainLibraryFile),
+            allModules = loadedIr.loadedFragments.values,
+            deserializer = loadedIr.linker,
+            configuration = compilerConfiguration,
+            dirtyFiles = loadedIr.loadedFragments.flatMap { (libFile, libFragment) ->
+                dirtyFileExports[libFile]?.let { libDirtyFiles ->
+                    libFragment.files.filter { file -> KotlinSourceFile(file) in libDirtyFiles }
+                } ?: emptyList()
+            },
+            exportedDeclarations = emptySet(),
+            mainArguments = mainArguments
+        )
+        eventCallback("updated files processing (lowering)")
+
+        val artifacts = buildModuleArtifactsAndCommitCache(loadedIr.linker, loadedIr.loadedFragments, rebuiltFragments)
+        eventCallback("cache committing")
+
+        return artifacts
+    }
+}
+
+// Used for tests only
+fun rebuildCacheForDirtyFiles(
+    library: KotlinLibrary,
+    configuration: CompilerConfiguration,
+    dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
+    dirtyFiles: Collection<String>?,
+    irFactory: IrFactory,
+    exportedDeclarations: Set<FqName>,
+    mainArguments: List<String>?,
+): Pair<IrModuleFragment, List<JsIrFragmentAndBinaryAst>> {
+    val emptyMetadata = object : KotlinSourceFileExports() {
+        override val inverseDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
+    }
+
+    val libFile = KotlinLibraryFile(library)
+    val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLibraryHeader(library).sourceFiles
+    val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })
+
+    val jsIrLoader = JsIrLinkerLoader(configuration, library, dependencyGraph, irFactory)
+    val (jsIrLinker, irModules) = jsIrLoader.loadIr(KotlinSourceFileMap<KotlinSourceFileExports>(modifiedFiles), true)
+
+    val currentIrModule = irModules[libFile] ?: notFoundIcError("loaded fragment", libFile)
+    val dirtyIrFiles = dirtyFiles?.let {
+        val files = it.toSet()
+        currentIrModule.files.filter { irFile -> irFile.fileEntry.name in files }
+    } ?: currentIrModule.files
+
+    return currentIrModule to buildCacheForModuleFiles(
+        mainModule = currentIrModule,
+        allModules = irModules.values,
+        deserializer = jsIrLinker,
+        configuration = configuration,
+        dirtyFiles = dirtyIrFiles,
+        exportedDeclarations = exportedDeclarations,
+        mainArguments = mainArguments
+    )
+}
+
+fun buildCacheForModuleFiles(
+    mainModule: IrModuleFragment,
+    allModules: Collection<IrModuleFragment>,
+    deserializer: JsIrLinker,
+    configuration: CompilerConfiguration,
+    dirtyFiles: Collection<IrFile>,
+    exportedDeclarations: Set<FqName>,
+    mainArguments: List<String>?
+): List<JsIrFragmentAndBinaryAst> {
+    return compileWithIC(
+        mainModule = mainModule,
+        allModules = allModules,
+        filesToLower = dirtyFiles,
+        configuration = configuration,
+        deserializer = deserializer,
+        mainArguments = mainArguments,
+        exportedDeclarations = exportedDeclarations,
+    )
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt

@@ -16,6 +16,8 @@ internal inline fun <T> File.ifExists(f: File.() -> T): T? = if (exists()) f() e
 internal fun File.recreate() {
     if (exists()) {
         delete()
+    } else {
+        parentFile?.mkdirs()
     }
     createNewFile()
 }
@@ -35,3 +37,23 @@ internal inline fun File.useCodedOutput(f: CodedOutputStream.() -> Unit) {
         out.flush()
     }
 }
+
+internal fun icError(msg: String): Nothing = error("IC internal error: $msg")
+
+internal fun notFoundIcError(what: String, libFile: KotlinLibraryFile? = null, srcFile: KotlinSourceFile? = null): Nothing {
+    val filePath = listOfNotNull(libFile?.path, srcFile?.path).joinToString(":") { File(it).name }
+    val msg = if (filePath.isEmpty()) what else "$what for $filePath"
+    icError("can not find $msg")
+}
+
+internal inline fun <E> buildListUntil(to: Int, builderAction: MutableList<E>.(Int) -> Unit): List<E> {
+    return buildList(to) { repeat(to) { builderAction(it) } }
+}
+
+internal inline fun <E> buildSetUntil(to: Int, builderAction: MutableSet<E>.(Int) -> Unit): Set<E> {
+    return buildSet(to) { repeat(to) { builderAction(it) } }
+}
+
+internal inline fun <K, V> buildMapUntil(to: Int, builderAction: MutableMap<K, V>.(Int) -> Unit): Map<K, V> {
+    return buildMap(to) { repeat(to) { builderAction(it) } }
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -6,311 +6,249 @@
 package org.jetbrains.kotlin.ir.backend.js.ic
 
 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
-import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
-import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
+import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrFragmentAndBinaryAst
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.javaFile
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
-import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import java.io.File
 
 
-class IncrementalCache(private val library: KotlinLibrary, cachePath: String) : ArtifactCache() {
+class IncrementalCache(private val library: KotlinLibrary, cachePath: String) {
     companion object {
-        private const val cacheFullInfoFile = "cache.full.info"
-        private const val cacheFastInfoFile = "cache.fast.info"
-        private const val binaryAstSuffix = "binary.ast"
+        private const val CACHE_HEADER = "ic.header.bin"
+
+        private const val BINARY_AST_SUFFIX = "ast.bin"
+        private const val METADATA_SUFFIX = "metadata.bin"
     }
 
-    class CacheFastInfo(
-        var moduleName: String? = null,
-        var flatHash: ICHash = ICHash(),
-        var transHash: ICHash = ICHash(),
-        var configHash: ICHash = ICHash(),
-        var initialFlatHash: ICHash = ICHash()
-    )
-
-    private enum class CacheState { NON_LOADED, FETCHED_FOR_DEPENDENCY, FETCHED_FULL }
-
-    private var state = CacheState.NON_LOADED
     private var forceRebuildJs = false
-
     private val cacheDir = File(cachePath)
-    private val signatureToIdMapping = mutableMapOf<String, Map<IdSignature, Int>>()
+    private val signatureToIndexMappingFromMetadata = mutableMapOf<KotlinSourceFile, MutableMap<IdSignature, Int>>()
 
-    private val fingerprints = mutableMapOf<String, ICHash>()
-    private val usedInlineFunctions = mutableMapOf<String, Map<IdSignature, ICHash>>()
-    private val implementedInlineFunctions = mutableMapOf<String, Map<IdSignature, ICHash>>()
+    private val libraryFile = KotlinLibraryFile(library)
 
-    private var cacheFastInfo = CacheFastInfo().apply {
-        File(cacheDir, cacheFastInfoFile).useCodedInputIfExists {
-            moduleName = readString()
-            flatHash = ICHash.fromProtoStream(this)
-            transHash = ICHash.fromProtoStream(this)
-            configHash = ICHash.fromProtoStream(this)
+    class CacheHeader(val klibFileHash: ICHash = ICHash(), val configHash: ICHash = ICHash()) {
+        fun toProtoStream(out: CodedOutputStream) {
+            klibFileHash.toProtoStream(out)
+            configHash.toProtoStream(out)
         }
-        initialFlatHash = flatHash
-    }
 
-    val srcFingerprints: Map<String, ICHash> get() = fingerprints
-    val usedFunctions: Map<String, Map<IdSignature, ICHash>> get() = usedInlineFunctions
-    val implementedFunctions: Collection<Map<IdSignature, ICHash>> get() = implementedInlineFunctions.values
-
-    val klibUpdated: Boolean get() = cacheFastInfo.run { initialFlatHash == ICHash() || initialFlatHash != flatHash }
-
-    val klibTransitiveHash: ICHash get() = cacheFastInfo.transHash
-
-    var srcFilesInOrderFromKLib: List<String> = emptyList()
-        private set
-
-    var deletedSrcFiles: Set<String> = emptySet()
-        private set
-
-    fun updateSignatureToIdMapping(srcPath: String, mapping: Map<IdSignature, Int>) {
-        signatureToIdMapping[srcPath] = mapping
-    }
-
-    fun updateHashes(
-        srcPath: String, fingerprint: ICHash, usedFunctions: Map<IdSignature, ICHash>?, implementedFunctions: Map<IdSignature, ICHash>?
-    ) {
-        fingerprints[srcPath] = fingerprint
-        usedFunctions?.let { usedInlineFunctions[srcPath] = it }
-        implementedFunctions?.let { implementedInlineFunctions[srcPath] = it }
-    }
-
-    fun invalidateCacheForNewConfig(configHash: ICHash) {
-        if (cacheFastInfo.configHash != configHash) {
-            invalidate()
-            cacheFastInfo.configHash = configHash
-        }
-    }
-
-    fun checkAndUpdateCacheFastInfo(flatHash: ICHash, transHash: ICHash): Boolean {
-        if (cacheFastInfo.transHash != transHash) {
-            cacheFastInfo.flatHash = flatHash
-            cacheFastInfo.transHash = transHash
-            return false
-        }
-        return true
-    }
-
-    private fun commitCacheFastInfo(klibModuleName: String? = null): Unit = cacheFastInfo.run {
-        moduleName = klibModuleName ?: moduleName
-        val name = moduleName ?: error("Internal error: uninitialized fast cache info for ${library.libraryName}")
-        File(cacheDir, cacheFastInfoFile).useCodedOutput {
-            writeStringNoTag(name)
-            flatHash.toProtoStream(this)
-            transHash.toProtoStream(this)
-            configHash.toProtoStream(this)
-        }
-    }
-
-    private fun CodedInputStream.readFunctionHashes(
-        deserializer: IdSignatureDeserializer, signatureToId: MutableMap<IdSignature, Int>? = null
-    ): Map<IdSignature, ICHash>? {
-        val functions = readInt32()
-        if (functions == 0) {
-            return null
-        }
-        val result = mutableMapOf<IdSignature, ICHash>()
-        for (funIndex in 0 until functions) {
-            val sigId = readInt32()
-            val hash = ICHash.fromProtoStream(this)
-            try {
-                val signature = deserializer.deserializeIdSignature(sigId)
-                result[signature] = hash
-                signatureToId?.let { it[signature] = sigId }
-            } catch (ex: IndexOutOfBoundsException) {
-                // Signature has been removed
+        companion object {
+            fun fromProtoStream(input: CodedInputStream): CacheHeader {
+                val klibFileHash = ICHash.fromProtoStream(input)
+                val configHash = ICHash.fromProtoStream(input)
+                return CacheHeader(klibFileHash, configHash)
             }
         }
-        return result
     }
 
-    fun fetchFullCacheData() {
-        when (state) {
-            CacheState.FETCHED_FULL -> return
-            CacheState.FETCHED_FOR_DEPENDENCY ->
-                error("Internal error: cache for ${library.libraryName} has been already fetched for dependency")
-            CacheState.NON_LOADED -> {
-                state = CacheState.FETCHED_FULL
-                val signatureReaders = library.filesAndSigReaders()
-                srcFilesInOrderFromKLib = signatureReaders.map { it.first }
+    private var cacheHeader = File(cacheDir, CACHE_HEADER).useCodedInputIfExists {
+        CacheHeader.fromProtoStream(this)
+    } ?: CacheHeader()
 
-                File(cacheDir, cacheFullInfoFile).useCodedInputIfExists {
-                    val deleted = mutableSetOf<String>()
+    private fun loadCachedFingerprints() = File(cacheDir, CACHE_HEADER).useCodedInputIfExists {
+        // skip cache header
+        CacheHeader.fromProtoStream(this@useCodedInputIfExists)
+        buildMapUntil(readInt32()) {
+            val file = KotlinSourceFile.fromProtoStream(this@useCodedInputIfExists)
+            put(file, ICHash.fromProtoStream(this@useCodedInputIfExists))
+        }
+    } ?: emptyMap()
 
-                    val signatureReadersMap = signatureReaders.toMap()
-                    val srcFiles = readInt32()
-                    for (srcIndex in 0 until srcFiles) {
-                        val srcPath = readString()
-                        val fingerprint = ICHash.fromProtoStream(this)
+    private val kotlinLibraryHeader: KotlinLibraryHeader by lazy { KotlinLibraryHeader(library) }
 
-                        val deserializer = signatureReadersMap[srcPath]
-                        if (deserializer != null) {
-                            fingerprints[srcPath] = fingerprint
-                            val signatureToId = mutableMapOf<IdSignature, Int>()
-                            readFunctionHashes(deserializer, signatureToId)?.let { implementedInlineFunctions[srcPath] = it }
-                            readFunctionHashes(deserializer, signatureToId)?.let { usedInlineFunctions[srcPath] = it }
-                            if (signatureToId.isNotEmpty()) {
-                                signatureToIdMapping[srcPath] = signatureToId
-                            }
+    private class KotlinSourceFileMetadataFromDisk(
+        override val inverseDependencies: KotlinSourceFileMap<Set<IdSignature>>,
+        override val directDependencies: KotlinSourceFileMap<Set<IdSignature>>,
+
+        override val importedInlineFunctions: Map<IdSignature, ICHash>
+    ) : KotlinSourceFileMetadata()
+
+    private object KotlinSourceFileMetadataNotExist : KotlinSourceFileMetadata() {
+        override val inverseDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
+        override val directDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
+
+        override val importedInlineFunctions = emptyMap<IdSignature, ICHash>()
+    }
+
+    private val kotlinLibrarySourceFileMetadata = mutableMapOf<KotlinSourceFile, KotlinSourceFileMetadata>()
+
+    private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix")
+
+    private fun commitCacheHeader(fingerprints: List<Pair<KotlinSourceFile, ICHash>>) = File(cacheDir, CACHE_HEADER).useCodedOutput {
+        cacheHeader.toProtoStream(this)
+        writeInt32NoTag(fingerprints.size)
+        for ((srcFile, fingerprint) in fingerprints) {
+            srcFile.toProtoStream(this)
+            fingerprint.toProtoStream(this)
+        }
+    }
+
+    fun buildModuleArtifactAndCommitCache(
+        moduleName: String,
+        rebuiltFileFragments: Map<KotlinSourceFile, JsIrFragmentAndBinaryAst>,
+        signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>
+    ): ModuleArtifact {
+        val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile ->
+            val binaryAstFile = srcFile.getCacheFile(BINARY_AST_SUFFIX)
+            val rebuiltFileFragment = rebuiltFileFragments[srcFile]
+            if (rebuiltFileFragment != null) {
+                binaryAstFile.apply { recreate() }.writeBytes(rebuiltFileFragment.binaryAst)
+            }
+
+            commitSourceFileMetadata(srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
+            SrcFileArtifact(srcFile.path, rebuiltFileFragment?.fragment, binaryAstFile)
+        }
+
+        return ModuleArtifact(moduleName, fileArtifacts, cacheDir, forceRebuildJs)
+    }
+
+    data class ModifiedFiles(
+        val modified: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
+        val removed: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
+        val newFiles: Set<KotlinSourceFile> = emptySet()
+    )
+
+    fun collectModifiedFiles(configHash: ICHash): ModifiedFiles {
+        val klibFileHash = library.libraryFile.javaFile().fileHashForIC()
+        cacheHeader = when {
+            cacheHeader.configHash != configHash -> {
+                cacheDir.deleteRecursively()
+                CacheHeader(klibFileHash, configHash)
+            }
+            cacheHeader.klibFileHash != klibFileHash -> CacheHeader(klibFileHash, configHash)
+            else -> return ModifiedFiles()
+        }
+
+        val cachedFingerprints = loadCachedFingerprints()
+        val deletedFiles = cachedFingerprints.keys.toMutableSet()
+        val newFiles = mutableSetOf<KotlinSourceFile>()
+
+        val newFingerprints = kotlinLibraryHeader.sourceFiles.mapIndexed { index, file -> file to library.fingerprint(index) }
+        val modifiedFiles = buildMap(newFingerprints.size) {
+            for ((file, fileNewFingerprint) in newFingerprints) {
+                val oldFingerprint = cachedFingerprints[file]
+                if (oldFingerprint == null) {
+                    newFiles += file
+                }
+                if (oldFingerprint != fileNewFingerprint) {
+                    val metadata = fetchSourceFileMetadata(file, false)
+                    put(file, metadata)
+                }
+                deletedFiles.remove(file)
+            }
+        }
+
+        val removedFilesMetadata = deletedFiles.associateWith {
+            val metadata = fetchSourceFileMetadata(it, false)
+            it.getCacheFile(BINARY_AST_SUFFIX).delete()
+            it.getCacheFile(METADATA_SUFFIX).delete()
+            metadata
+        }
+
+        forceRebuildJs = deletedFiles.isNotEmpty()
+        commitCacheHeader(newFingerprints)
+
+        return ModifiedFiles(modifiedFiles, removedFilesMetadata, newFiles)
+    }
+
+    fun fetchSourceFileFullMetadata(srcFile: KotlinSourceFile): KotlinSourceFileMetadata {
+        return fetchSourceFileMetadata(srcFile, true)
+    }
+
+    fun updateSourceFileMetadata(srcFile: KotlinSourceFile, sourceFileMetadata: KotlinSourceFileMetadata) {
+        kotlinLibrarySourceFileMetadata[srcFile] = sourceFileMetadata
+    }
+
+    private fun fetchSourceFileMetadata(srcFile: KotlinSourceFile, loadSignatures: Boolean) =
+        kotlinLibrarySourceFileMetadata.getOrPut(srcFile) {
+            val signatureToIndexMapping = signatureToIndexMappingFromMetadata.getOrPut(srcFile) { mutableMapOf() }
+            fun IdSignatureDeserializer.deserializeIdSignatureAndSave(index: Int): IdSignature {
+                val signature = deserializeIdSignature(index)
+                signatureToIndexMapping[signature] = index
+                return signature
+            }
+
+            val deserializer: IdSignatureDeserializer by lazy {
+                kotlinLibraryHeader.signatureDeserializers[srcFile] ?: notFoundIcError("signature deserializer", libraryFile, srcFile)
+            }
+
+            srcFile.getCacheFile(METADATA_SUFFIX).useCodedInputIfExists {
+                fun readDependencies() = buildMapUntil(readInt32()) {
+                    val libraryFile = KotlinLibraryFile.fromProtoStream(this@useCodedInputIfExists)
+                    val depends = buildMapUntil(readInt32()) {
+                        val dependencySrcFile = KotlinSourceFile.fromProtoStream(this@useCodedInputIfExists)
+                        val dependencySignatures = if (loadSignatures) {
+                            buildSetUntil(readInt32()) { add(deserializer.deserializeIdSignatureAndSave(readInt32())) }
                         } else {
-                            deleted.add(srcPath)
-                            skipFunctionHashes()
-                            skipFunctionHashes()
+                            repeat(readInt32()) { readInt32() }
+                            emptySet()
+                        }
+                        put(dependencySrcFile, dependencySignatures)
+                    }
+                    put(libraryFile, depends)
+                }
+
+                val directDependencies = KotlinSourceFileMap(readDependencies())
+                val reverseDependencies = KotlinSourceFileMap(readDependencies())
+
+                val importedInlineFunctions = if (loadSignatures) {
+                    buildMapUntil(readInt32()) {
+                        val signature = deserializer.deserializeIdSignatureAndSave(readInt32())
+                        val transitiveHash = ICHash.fromProtoStream(this@useCodedInputIfExists)
+                        put(signature, transitiveHash)
+                    }
+                } else {
+                    emptyMap()
+                }
+
+                KotlinSourceFileMetadataFromDisk(reverseDependencies, directDependencies, importedInlineFunctions)
+            } ?: KotlinSourceFileMetadataNotExist
+        }
+
+    private fun commitSourceFileMetadata(srcFile: KotlinSourceFile, signatureToIndexMapping: Map<IdSignature, Int>) {
+        val headerCacheFile = srcFile.getCacheFile(METADATA_SUFFIX)
+        val sourceFileMetadata = kotlinLibrarySourceFileMetadata[srcFile] ?: notFoundIcError("metadata", libraryFile, srcFile)
+        if (sourceFileMetadata.isEmpty()) {
+            headerCacheFile.delete()
+            return
+        }
+        if (sourceFileMetadata is KotlinSourceFileMetadataFromDisk) {
+            return
+        }
+
+        val signatureToIndexMappingSaved = signatureToIndexMappingFromMetadata[srcFile] ?: emptyMap()
+        fun serializeSignature(signature: IdSignature): Int {
+            val index = signatureToIndexMapping[signature] ?: signatureToIndexMappingSaved[signature]
+            return index ?: notFoundIcError("signature $signature", libraryFile, srcFile)
+        }
+
+        headerCacheFile.useCodedOutput {
+            fun writeDepends(depends: KotlinSourceFileMap<Set<IdSignature>>) {
+                writeInt32NoTag(depends.size)
+                for ((dependencyLibFile, dependencySrcFiles) in depends) {
+                    dependencyLibFile.toProtoStream(this)
+                    writeInt32NoTag(dependencySrcFiles.size)
+                    for ((dependencySrcFile, signatures) in dependencySrcFiles) {
+                        dependencySrcFile.toProtoStream(this)
+                        writeInt32NoTag(signatures.size)
+                        for (signature in signatures) {
+                            writeInt32NoTag(serializeSignature(signature))
                         }
                     }
-
-                    deletedSrcFiles = deleted
                 }
             }
-        }
-    }
 
-    private fun CodedInputStream.skipFunctionHashes() {
-        val functions = readInt32()
-        for (funIndex in 0 until functions) {
-            readInt32() // skip sigid
-            readFixed64() // skip hash
-        }
-    }
+            writeDepends(sourceFileMetadata.directDependencies)
+            writeDepends(sourceFileMetadata.inverseDependencies)
 
-    fun fetchCacheDataForDependency() = File(cacheDir, cacheFullInfoFile).useCodedInputIfExists {
-        if (state == CacheState.NON_LOADED) {
-            state = CacheState.FETCHED_FOR_DEPENDENCY
-            val signatureReadersMap = library.filesAndSigReaders().toMap()
-
-            val srcFiles = readInt32()
-            for (srcIndex in 0 until srcFiles) {
-                val srcPath = readString()
-                val fingerprint = ICHash.fromProtoStream(this)
-
-                val deserializer = signatureReadersMap[srcPath]
-                if (deserializer != null) {
-                    fingerprints[srcPath] = fingerprint
-                    readFunctionHashes(deserializer)?.let { implementedInlineFunctions[srcPath] = it }
-                }
-                skipFunctionHashes()
+            writeInt32NoTag(sourceFileMetadata.importedInlineFunctions.size)
+            for ((signature, transitiveHash) in sourceFileMetadata.importedInlineFunctions) {
+                writeInt32NoTag(serializeSignature(signature))
+                transitiveHash.toProtoStream(this)
             }
         }
     }
-
-    private fun getBinaryAstPath(srcFile: String): File {
-        val binaryAstFileName = "${File(srcFile).name}.${srcFile.stringHashForIC()}.$binaryAstSuffix"
-        return File(cacheDir, binaryAstFileName)
-    }
-
-    private fun CodedOutputStream.writeFunctionHashes(sigToIndexMap: Map<IdSignature, Int>, hashes: Map<IdSignature, ICHash>) {
-        writeInt32NoTag(hashes.size)
-        for ((sig, functionHash) in hashes) {
-            val sigId = sigToIndexMap[sig] ?: error("No index found for sig $sig")
-            writeInt32NoTag(sigId)
-            functionHash.toProtoStream(this)
-        }
-    }
-
-    private fun commitCacheFullInfo() {
-        if (state != CacheState.FETCHED_FULL) {
-            error("Internal error: cache for ${library.libraryName} has not been fetched fully")
-        }
-
-        File(cacheDir, cacheFullInfoFile).useCodedOutput {
-            writeInt32NoTag(fingerprints.size)
-            for ((srcPath, fingerprint) in fingerprints) {
-                writeStringNoTag(srcPath)
-                fingerprint.toProtoStream(this)
-
-                val sigToIndexMap = signatureToIdMapping[srcPath] ?: emptyMap()
-                writeFunctionHashes(sigToIndexMap, implementedInlineFunctions[srcPath] ?: emptyMap())
-                writeFunctionHashes(sigToIndexMap, usedInlineFunctions[srcPath] ?: emptyMap())
-            }
-        }
-    }
-
-    private fun clearCacheAfterCommit() {
-        state = CacheState.FETCHED_FOR_DEPENDENCY
-        forceRebuildJs = deletedSrcFiles.isNotEmpty()
-        signatureToIdMapping.clear()
-        usedInlineFunctions.clear()
-        srcFilesInOrderFromKLib = emptyList()
-        deletedSrcFiles = emptySet()
-        binaryAsts.clear()
-    }
-
-    fun commitCacheForRemovedSrcFiles() {
-        commitCacheFastInfo()
-        if (deletedSrcFiles.isNotEmpty()) {
-            commitCacheFullInfo()
-        }
-        clearCacheAfterCommit()
-    }
-
-    fun commitCacheForRebuiltSrcFiles(klibModuleName: String) {
-        commitCacheFastInfo(klibModuleName)
-        commitCacheFullInfo()
-        for ((srcPath, ast) in binaryAsts) {
-            getBinaryAstPath(srcPath).apply { recreate() }.writeBytes(ast)
-        }
-        clearCacheAfterCommit()
-    }
-
-    override fun fetchArtifacts() = ModuleArtifact(
-        moduleName = cacheFastInfo.moduleName ?: error("Internal error: missing module name"),
-        fileArtifacts = fingerprints.keys.map {
-            SrcFileArtifact(it, fragments[it], getBinaryAstPath(it))
-        },
-        artifactsDir = cacheDir,
-        forceRebuildJs = forceRebuildJs
-    )
-
-    fun invalidate() {
-        cacheDir.deleteRecursively()
-        signatureToIdMapping.clear()
-        implementedInlineFunctions.clear()
-        usedInlineFunctions.clear()
-        fingerprints.clear()
-        binaryAsts.clear()
-        fragments.clear()
-        cacheFastInfo = CacheFastInfo()
-        srcFilesInOrderFromKLib = emptyList()
-        deletedSrcFiles = emptySet()
-    }
-
-    fun invalidateForSrcFile(srcPath: String) {
-        getBinaryAstPath(srcPath).delete()
-        signatureToIdMapping.remove(srcPath)
-        implementedInlineFunctions.remove(srcPath)
-        usedInlineFunctions.remove(srcPath)
-        fingerprints.remove(srcPath)
-        binaryAsts.remove(srcPath)
-        fragments.remove(srcPath)
-    }
-
-    private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDeserializer>> {
-        val fileSize = fileCount()
-        val result = ArrayList<Pair<String, IdSignatureDeserializer>>(fileSize)
-        val extReg = ExtensionRegistryLite.newInstance()
-
-        for (i in 0 until fileSize) {
-            val fileStream = file(i).codedInputStream
-            val fileProto = IrFile.parseFrom(fileStream, extReg)
-            val sigReader = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
-                private fun err(): Nothing = error("Not supported")
-                override fun irDeclaration(index: Int): ByteArray = err()
-                override fun type(index: Int): ByteArray = err()
-                override fun signature(index: Int): ByteArray = signature(index, i)
-                override fun string(index: Int): ByteArray = string(index, i)
-                override fun body(index: Int): ByteArray = err()
-                override fun debugInfo(index: Int): ByteArray? = null
-            }), null)
-
-            result.add(fileProto.fileEntry.name to sigReader)
-        }
-
-        return result
-    }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/InlineFunctionTransitiveHashCalculator.kt

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrFunction
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
+import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.expressions.IrFunctionReference
+import org.jetbrains.kotlin.ir.util.IdSignature
+import org.jetbrains.kotlin.ir.util.isFakeOverride
+import org.jetbrains.kotlin.ir.util.render
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
+import org.jetbrains.kotlin.ir.visitors.acceptVoid
+
+
+class InlineFunctionTransitiveHashCalculator {
+    private val flatHashes = mutableMapOf<IrFunction, ICHash>()
+    private val inlineFunctionCallGraph: MutableMap<IrFunction, Set<IrFunction>> = mutableMapOf()
+    private val processingFunctions = mutableSetOf<IrFunction>()
+    private val functionTransitiveHashes = mutableMapOf<IrFunction, ICHash>()
+    private val idSignatureTransitiveHashes = mutableMapOf<IdSignature, ICHash>()
+
+    val transitiveHashes: Map<IdSignature, ICHash>
+        get() = idSignatureTransitiveHashes
+
+    private inner class FlatHashCalculator : IrElementVisitorVoid {
+        override fun visitElement(element: IrElement) = element.acceptChildren(this, null)
+
+        override fun visitSimpleFunction(declaration: IrSimpleFunction) {
+            if (declaration.isInline) {
+                if (declaration in flatHashes) {
+                    return
+                }
+                flatHashes[declaration] = declaration.irElementHashForIC()
+            }
+            // go deeper since local inline special declarations (like a reference adaptor) may appear
+            declaration.acceptChildren(this, null)
+        }
+    }
+
+    private inner class CallGraphBuilder : IrElementVisitor<Unit, MutableSet<IrFunction>> {
+        var inlineFunctionCallDepth: Int = 0
+
+        override fun visitElement(element: IrElement, data: MutableSet<IrFunction>) = element.acceptChildren(this, data)
+
+        override fun visitSimpleFunction(declaration: IrSimpleFunction, data: MutableSet<IrFunction>) {
+            if (declaration in inlineFunctionCallGraph) {
+                return
+            }
+            val newGraph = mutableSetOf<IrFunction>()
+            inlineFunctionCallGraph[declaration] = newGraph
+            declaration.acceptChildren(this, newGraph)
+        }
+
+        override fun visitCall(expression: IrCall, data: MutableSet<IrFunction>) {
+            val callee = expression.symbol.owner
+            if (callee.isInline) {
+                // TODO: do not ignore fake overrides after KT-51896
+                if (!callee.isFakeOverride) {
+                    data += callee
+                }
+                inlineFunctionCallDepth += 1
+            }
+            expression.acceptChildren(this, data)
+            if (callee.isInline) {
+                inlineFunctionCallDepth -= 1
+                if (inlineFunctionCallDepth < 0) {
+                    icError("inline function calls depth inconsistent")
+                }
+            }
+        }
+
+        override fun visitFunctionReference(expression: IrFunctionReference, data: MutableSet<IrFunction>) {
+            val reference = expression.symbol.owner
+            if (inlineFunctionCallDepth > 0 && reference.isInline) {
+                // this if is fine, because fake overrides are not inlined as function reference calls even as inline function args
+                if (!reference.isFakeOverride) {
+                    data += reference
+                }
+            }
+            expression.acceptChildren(this, data)
+        }
+    }
+
+    private fun getInlineFunctionTransitiveHash(f: IrFunction): ICHash = functionTransitiveHashes.getOrPut(f) {
+        if (!processingFunctions.add(f)) {
+            icError("inline circle through function ${f.render()} detected")
+        }
+        val callees = inlineFunctionCallGraph[f] ?: icError("inline function is missed in inline graph ${f.render()}")
+        val flatHash = flatHashes[f] ?: icError("no flat hash for ${f.render()}")
+        var functionInlineHash = flatHash
+        for (callee in callees) {
+            functionInlineHash = functionInlineHash.combineWith(getInlineFunctionTransitiveHash(callee))
+        }
+        processingFunctions.remove(f)
+        f.symbol.signature?.let { idSignatureTransitiveHashes[it] = functionInlineHash }
+        functionInlineHash
+    }
+
+    private fun updateTransitiveHashesByCallGraph() {
+        for ((f, callees) in inlineFunctionCallGraph.entries) {
+            if (f.isInline) {
+                getInlineFunctionTransitiveHash(f)
+            } else {
+                callees.forEach(::getInlineFunctionTransitiveHash)
+            }
+        }
+    }
+
+    fun updateTransitiveHashes(fragments: Collection<IrModuleFragment>) {
+        fragments.forEach { it.acceptVoid(FlatHashCalculator()) }
+        fragments.forEach { it.acceptChildren(CallGraphBuilder(), mutableSetOf()) }
+        updateTransitiveHashesByCallGraph()
+    }
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/InlineGraphTracker.kt

@@ -1,159 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.ic
-
-import org.jetbrains.kotlin.ir.IrElement
-import org.jetbrains.kotlin.ir.declarations.IrFile
-import org.jetbrains.kotlin.ir.declarations.IrFunction
-import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
-import org.jetbrains.kotlin.ir.expressions.IrCall
-import org.jetbrains.kotlin.ir.expressions.IrFunctionReference
-import org.jetbrains.kotlin.ir.util.IdSignature
-import org.jetbrains.kotlin.ir.util.file
-import org.jetbrains.kotlin.ir.util.isFakeOverride
-import org.jetbrains.kotlin.ir.util.render
-import org.jetbrains.kotlin.ir.visitors.IrElementVisitor
-import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
-
-
-class InlineFunctionFlatHashBuilder : IrElementVisitorVoid {
-    override fun visitElement(element: IrElement) {
-        element.acceptChildren(this, null)
-    }
-
-    override fun visitSimpleFunction(declaration: IrSimpleFunction) {
-        if (declaration.isInline) {
-            flatHashes[declaration] = declaration.irElementHashForIC()
-        }
-        // go deeper since local inline special declarations (like a reference adaptor) may appear
-        declaration.acceptChildren(this, null)
-    }
-
-    private val flatHashes = mutableMapOf<IrFunction, ICHash>()
-
-    fun getFlatHashes() = flatHashes
-}
-
-interface InlineFunctionHashProvider {
-    fun hashForExternalFunction(declaration: IrFunction): ICHash?
-}
-
-class InlineFunctionHashBuilder(
-    private val hashProvider: InlineFunctionHashProvider,
-    private val flatHashes: Map<IrFunction, ICHash>
-) {
-    private val inlineFunctionCallGraph: MutableMap<IrFunction, Set<IrFunction>> = mutableMapOf()
-
-    private inner class GraphBuilder : IrElementVisitor<Unit, MutableSet<IrFunction>> {
-        var inlineFunctionCallDepth: Int = 0
-
-        override fun visitElement(element: IrElement, data: MutableSet<IrFunction>) {
-            element.acceptChildren(this, data)
-        }
-
-        override fun visitSimpleFunction(declaration: IrSimpleFunction, data: MutableSet<IrFunction>) {
-            val newGraph = mutableSetOf<IrFunction>()
-            inlineFunctionCallGraph[declaration] = newGraph
-            declaration.acceptChildren(this, newGraph)
-        }
-
-        override fun visitCall(expression: IrCall, data: MutableSet<IrFunction>) {
-            val callee = expression.symbol.owner
-            if (callee.isInline) {
-                // TODO: do not ignore fake overides after KT-51896
-                if (!callee.isFakeOverride) {
-                    data += callee
-                }
-                inlineFunctionCallDepth += 1
-            }
-            expression.acceptChildren(this, data)
-            if (callee.isInline) {
-                inlineFunctionCallDepth -= 1
-                if (inlineFunctionCallDepth < 0) {
-                    error("Internal error: inline function calls depth inconsistency")
-                }
-            }
-        }
-
-        override fun visitFunctionReference(expression: IrFunctionReference, data: MutableSet<IrFunction>) {
-            val reference = expression.symbol.owner
-            if (inlineFunctionCallDepth > 0 && reference.isInline) {
-                // this if is fine, because fake overrides are not inlined as function reference calls even as inline function args
-                if (!reference.isFakeOverride) {
-                    data += reference
-                }
-            }
-            expression.acceptChildren(this, data)
-        }
-    }
-
-    private inner class InlineFunctionHashProcessor {
-        private val computedHashes = mutableMapOf<IrFunction, ICHash>()
-        private val processingFunctions = mutableSetOf<IrFunction>()
-
-        private fun processInlineFunction(f: IrFunction): ICHash = computedHashes.getOrPut(f) {
-            if (!processingFunctions.add(f)) {
-                error("Inline circle through function ${f.render()} detected")
-            }
-            val callees = inlineFunctionCallGraph[f] ?: error("Internal error: Inline function is missed in inline graph ${f.render()}")
-            val flatHash = flatHashes[f] ?: error("Internal error: No flat hash for ${f.render()}")
-            var functionInlineHash = flatHash
-            for (callee in callees) {
-                functionInlineHash = functionInlineHash.combineWith(processCallee(callee))
-            }
-            processingFunctions.remove(f)
-            functionInlineHash
-        }
-
-        private fun processCallee(callee: IrFunction): ICHash {
-            if (callee in flatHashes) {
-                return processInlineFunction(callee)
-            }
-            return hashProvider.hashForExternalFunction(callee) ?: error("Internal error: No hash found for ${callee.render()}")
-        }
-
-        fun process(): Map<IrFunction, ICHash> {
-            for ((f, callees) in inlineFunctionCallGraph.entries) {
-                if (f.isInline) {
-                    processInlineFunction(f)
-                } else {
-                    callees.forEach(::processCallee)
-                }
-            }
-            return computedHashes
-        }
-    }
-
-    fun buildHashes(dirtyFiles: Collection<IrFile>): Map<IrFunction, ICHash> {
-        dirtyFiles.forEach { it.acceptChildren(GraphBuilder(), mutableSetOf()) }
-        return InlineFunctionHashProcessor().process()
-    }
-
-    fun buildInlineGraph(computedHashed: Map<IrFunction, ICHash>): Map<IrFile, Map<IdSignature, ICHash>> {
-        val perFileInlineGraph = inlineFunctionCallGraph.entries.groupBy({ it.key.file }) {
-            it.value
-        }
-
-        return perFileInlineGraph.entries.associate {
-            val usedInlineFunctions = mutableMapOf<IdSignature, ICHash>()
-            it.value.forEach { edges ->
-                edges.forEach { callee ->
-                    // TODO: do not ignore fake overides after KT-51896
-                    if (!callee.isFakeOverride) {
-                        val signature = callee.symbol.signature
-                        if (signature?.visibleCrossFile == true) {
-                            val calleeHash = computedHashed[callee]
-                                ?: hashProvider.hashForExternalFunction(callee)
-                                ?: error("Internal error: No hash found for ${callee.render()}")
-                            usedInlineFunctions[signature] = calleeHash
-                        }
-                    }
-                }
-            }
-            it.key to usedInlineFunctions
-        }
-    }
-}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsExecutableProducer.kt

@@ -53,8 +53,8 @@ class JsExecutableProducer(
                 }
                 jsIrHeader.associatedModule = artifact.loadJsIrModule()
             }
-            val associatedModule = jsIrHeader.associatedModule ?: error("Internal error: cannot load module $moduleName")
-            val crossRef = crossModuleReferences[jsIrHeader] ?: error("Internal error: cannot find cross references for module $moduleName")
+            val associatedModule = jsIrHeader.associatedModule ?: icError("can not load module $moduleName")
+            val crossRef = crossModuleReferences[jsIrHeader] ?: icError("can not find cross references for module $moduleName")
             crossRef.initJsImportsForModule(associatedModule)
 
             val compiledModule = generateSingleWrappedModuleBody(

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
+import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy
 import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
@@ -71,29 +72,46 @@ internal class JsIrLinkerLoader(
         return dependencyGraph.keys.associateBy { klib -> getModuleDescriptor(klib) }
     }
 
-    fun processJsIrLinker(dirtyFiles: Collection<String>?): Triple<JsIrLinker, IrModuleFragment, Collection<IrModuleFragment>> {
+    data class LoadedJsIr(val linker: JsIrLinker, val loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>)
+
+    fun loadIr(modifiedFiles: KotlinSourceFileMap<KotlinSourceFileExports>, loadAllIr: Boolean = false): LoadedJsIr {
         val loadedModules = loadModules()
         val jsIrLinker = createLinker(loadedModules)
-        val irModules = ArrayList<Pair<IrModuleFragment, KotlinLibrary>>(loadedModules.size)
 
-        // TODO: modules deserialized here have to be reused for cache building further
-        for ((descriptor, loadedLibrary) in loadedModules) {
-            if (library == loadedLibrary) {
-                if (dirtyFiles != null) {
-                    irModules.add(jsIrLinker.deserializeDirtyFiles(descriptor, loadedLibrary, dirtyFiles) to loadedLibrary)
-                } else {
-                    irModules.add(jsIrLinker.deserializeFullModule(descriptor, loadedLibrary) to loadedLibrary)
-                }
-            } else {
-                irModules.add(jsIrLinker.deserializeHeadersWithInlineBodies(descriptor, loadedLibrary) to loadedLibrary)
+        val irModules = loadedModules.entries.associate { (descriptor, module) ->
+            val libraryFile = KotlinLibraryFile(module)
+            val modifiedStrategy = when {
+                loadAllIr -> DeserializationStrategy.ALL
+                module == library -> DeserializationStrategy.ALL
+                else -> DeserializationStrategy.EXPLICITLY_EXPORTED
             }
+            val modified = modifiedFiles[libraryFile] ?: emptyMap()
+            libraryFile to jsIrLinker.deserializeIrModuleHeader(descriptor, module, {
+                when (KotlinSourceFile(it)) {
+                    in modified -> modifiedStrategy
+                    else -> DeserializationStrategy.WITH_INLINE_BODIES
+                }
+            })
         }
 
         jsIrLinker.init(null, emptyList())
+
+        if (!loadAllIr) {
+            for ((loadingLibFile, loadingSrcFiles) in modifiedFiles) {
+                val loadingIrModule = irModules[loadingLibFile] ?: notFoundIcError("loading fragment", loadingLibFile)
+                val moduleDeserializer = jsIrLinker.moduleDeserializer(loadingIrModule.descriptor)
+                for (loadingSrcFileSignatures in loadingSrcFiles.values) {
+                    for (loadingSignature in loadingSrcFileSignatures.getExportedSignatures()) {
+                        if (loadingSignature in moduleDeserializer) {
+                            moduleDeserializer.addModuleReachableTopLevel(loadingSignature)
+                        }
+                    }
+                }
+            }
+        }
+
         ExternalDependenciesGenerator(jsIrLinker.symbolTable, listOf(jsIrLinker)).generateUnboundSymbolsAsDependencies()
         jsIrLinker.postProcess()
-
-        val currentIrModule = irModules.find { it.second == library }?.first!!
-        return Triple(jsIrLinker, currentIrModule, irModules.map { it.first })
+        return LoadedJsIr(jsIrLinker, irModules)
     }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsMultiModuleCache.kt

@@ -12,9 +12,9 @@ import java.io.File
 
 class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
     companion object {
-        private const val cacheModuleHeader = "cache.module.header.info"
-        private const val cacheJsModuleFile = "cache.module.js"
-        private const val cacheJsMapModuleFile = "cache.module.js.map"
+        private const val JS_MODULE_HEADER = "js.module.header.bin"
+        private const val CACHED_MODULE_JS = "module.js"
+        private const val CACHED_MODULE_JS_MAP = "module.js.map"
     }
 
     private enum class NameType(val typeMask: Int) {
@@ -25,7 +25,7 @@ class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
 
     private val headerToCachedInfo = mutableMapOf<JsIrModuleHeader, CachedModuleInfo>()
 
-    private fun ModuleArtifact.fetchModuleInfo() = File(artifactsDir, cacheModuleHeader).useCodedInputIfExists {
+    private fun ModuleArtifact.fetchModuleInfo() = File(artifactsDir, JS_MODULE_HEADER).useCodedInputIfExists {
         val definitions = mutableSetOf<String>()
         val nameBindings = mutableMapOf<String, String>()
 
@@ -49,7 +49,7 @@ class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
     }
 
     private fun CachedModuleInfo.commitModuleInfo() = artifact.artifactsDir?.let { cacheDir ->
-        File(cacheDir, cacheModuleHeader).useCodedOutput {
+        File(cacheDir, JS_MODULE_HEADER).useCodedOutput {
             val names = mutableMapOf<String, Pair<Int, String?>>()
             for ((tag, name) in jsIrHeader.nameBindings) {
                 names[tag] = NameType.NAME_BINDINGS.typeMask to name
@@ -72,15 +72,15 @@ class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
     }
 
     fun fetchCompiledJsCode(artifact: ModuleArtifact) = artifact.artifactsDir?.let { cacheDir ->
-        val jsCode = File(cacheDir, cacheJsModuleFile).ifExists { readText() }
-        val sourceMap = File(cacheDir, cacheJsMapModuleFile).ifExists { readText() }
+        val jsCode = File(cacheDir, CACHED_MODULE_JS).ifExists { readText() }
+        val sourceMap = File(cacheDir, CACHED_MODULE_JS_MAP).ifExists { readText() }
         jsCode?.let { CompilationOutputs(it, null, sourceMap) }
     }
 
     fun commitCompiledJsCode(artifact: ModuleArtifact, compilationOutputs: CompilationOutputs) = artifact.artifactsDir?.let { cacheDir ->
-        val jsCodeCache = File(cacheDir, cacheJsModuleFile).apply { recreate() }
+        val jsCodeCache = File(cacheDir, CACHED_MODULE_JS).apply { recreate() }
         jsCodeCache.writeText(compilationOutputs.jsCode)
-        val jsMapCache = File(cacheDir, cacheJsMapModuleFile)
+        val jsMapCache = File(cacheDir, CACHED_MODULE_JS_MAP)
         if (compilationOutputs.sourceMap != null) {
             jsMapCache.recreate()
             jsMapCache.writeText(compilationOutputs.sourceMap)
@@ -104,7 +104,7 @@ class JsMultiModuleCache(private val moduleArtifacts: List<ModuleArtifact>) {
 
     fun loadRequiredJsIrModules(crossModuleReferences: Map<JsIrModuleHeader, CrossModuleReferences>) {
         for ((header, references) in crossModuleReferences) {
-            val cachedInfo = headerToCachedInfo[header] ?: error("Internal error: cannot find artifact for module ${header.moduleName}")
+            val cachedInfo = headerToCachedInfo[header] ?: notFoundIcError("artifact for module ${header.moduleName}")
             val actualCrossModuleHash = references.crossModuleReferencesHashForIC()
             if (header.associatedModule == null && cachedInfo.crossModuleReferencesHash != actualCrossModuleHash) {
                 header.associatedModule = cachedInfo.artifact.loadJsIrModule()

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt

@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
+import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
+import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
+import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
+
+internal class KotlinLibraryHeader(library: KotlinLibrary) {
+    val sourceFiles: List<KotlinSourceFile>
+    val signatureDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer>
+
+    init {
+        val filesCount = library.fileCount()
+        val extReg = ExtensionRegistryLite.newInstance()
+        val files = buildListUntil(filesCount) {
+            val fileProto = IrFile.parseFrom(library.file(it).codedInputStream, extReg)
+            add(KotlinSourceFile(fileProto.fileEntry.name))
+        }
+
+        val deserializers = buildMapUntil(filesCount) {
+            put(files[it], IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
+                private fun err(): Nothing = icError("Not supported")
+                override fun irDeclaration(index: Int): ByteArray = err()
+                override fun type(index: Int): ByteArray = err()
+                override fun signature(index: Int): ByteArray = library.signature(index, it)
+                override fun string(index: Int): ByteArray = library.string(index, it)
+                override fun body(index: Int): ByteArray = err()
+                override fun debugInfo(index: Int): ByteArray? = null
+            }), null))
+        }
+
+        sourceFiles = files
+        signatureDeserializers = deserializers
+    }
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinSourceFileMetadata.kt

@@ -0,0 +1,102 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js.ic
+
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.util.IdSignature
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.protobuf.CodedInputStream
+import org.jetbrains.kotlin.protobuf.CodedOutputStream
+
+@JvmInline
+value class KotlinLibraryFile(val path: String) {
+    constructor(lib: KotlinLibrary) : this(lib.libraryFile.canonicalPath)
+
+    fun toProtoStream(out: CodedOutputStream) = out.writeStringNoTag(path)
+
+    companion object {
+        fun fromProtoStream(input: CodedInputStream) = KotlinLibraryFile(input.readString())
+    }
+}
+
+@JvmInline
+value class KotlinSourceFile(val path: String) {
+    constructor(irFile: IrFile) : this(irFile.fileEntry.name)
+
+    fun toProtoStream(out: CodedOutputStream) = out.writeStringNoTag(path)
+
+    companion object {
+        fun fromProtoStream(input: CodedInputStream) = KotlinSourceFile(input.readString())
+    }
+}
+
+open class KotlinSourceFileMap<out T>(files: Map<KotlinLibraryFile, Map<KotlinSourceFile, T>>) :
+    Map<KotlinLibraryFile, Map<KotlinSourceFile, T>> by files {
+
+    inline fun forEachFile(f: (KotlinLibraryFile, KotlinSourceFile, T) -> Unit) =
+        forEach { (lib, files) -> files.forEach { (file, data) -> f(lib, file, data) } }
+
+    operator fun get(libFile: KotlinLibraryFile, sourceFile: KotlinSourceFile): T? = get(libFile)?.get(sourceFile)
+}
+
+class KotlinSourceFileMutableMap<T>(
+    private val files: MutableMap<KotlinLibraryFile, MutableMap<KotlinSourceFile, T>> = mutableMapOf()
+) : KotlinSourceFileMap<T>(files) {
+
+    operator fun set(libFile: KotlinLibraryFile, sourceFile: KotlinSourceFile, data: T) = getOrPutFiles(libFile).put(sourceFile, data)
+    operator fun set(libFile: KotlinLibraryFile, sourceFiles: MutableMap<KotlinSourceFile, T>) = files.put(libFile, sourceFiles)
+
+    fun getOrPutFiles(libFile: KotlinLibraryFile) = files.getOrPut(libFile) { mutableMapOf() }
+
+    fun copyFilesFrom(other: KotlinSourceFileMap<T>) {
+        for ((libFile, srcFiles) in other) {
+            files.getOrPut(libFile) { mutableMapOf() } += srcFiles
+        }
+    }
+
+    fun removeFile(libFile: KotlinLibraryFile, sourceFile: KotlinSourceFile) {
+        val libFiles = files[libFile]
+        if (libFiles != null) {
+            libFiles.remove(sourceFile)
+            if (libFiles.isEmpty()) {
+                files.remove(libFile)
+            }
+        }
+    }
+
+    fun clear() = files.clear()
+}
+
+fun <T> KotlinSourceFileMap<T>.toMutable(): KotlinSourceFileMutableMap<T> {
+    return KotlinSourceFileMutableMap(entries.associateTo(mutableMapOf()) { it.key to it.value.toMutableMap() })
+}
+
+fun KotlinSourceFileMap<Set<IdSignature>>.flatSignatures(): Set<IdSignature> {
+    val allSignatures = mutableSetOf<IdSignature>()
+    forEachFile { _, _, signatures -> allSignatures += signatures }
+    return allSignatures
+}
+
+fun KotlinSourceFileMutableMap<MutableSet<IdSignature>>.addSignature(
+    lib: KotlinLibraryFile, src: KotlinSourceFile, signature: IdSignature
+) = when (val signatures = this[lib, src]) {
+    null -> this[lib, src] = mutableSetOf(signature)
+    else -> signatures += signature
+}
+
+abstract class KotlinSourceFileExports {
+    abstract val inverseDependencies: KotlinSourceFileMap<Set<IdSignature>>
+
+    open fun getExportedSignatures(): Set<IdSignature> = inverseDependencies.flatSignatures()
+}
+
+abstract class KotlinSourceFileMetadata : KotlinSourceFileExports() {
+    abstract val directDependencies: KotlinSourceFileMap<Set<IdSignature>>
+
+    abstract val importedInlineFunctions: Map<IdSignature, ICHash>
+}
+
+fun KotlinSourceFileMetadata.isEmpty() = inverseDependencies.isEmpty() && directDependencies.isEmpty() && importedInlineFunctions.isEmpty()

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ModuleArtifact.kt

@@ -41,18 +41,3 @@ class ModuleArtifact(
         return JsIrModule(moduleSafeName, moduleSafeName, fragments)
     }
 }
-
-abstract class ArtifactCache {
-    protected val binaryAsts = mutableMapOf<String, ByteArray>()
-    protected val fragments = mutableMapOf<String, JsIrProgramFragment>()
-
-    fun saveBinaryAst(srcPath: String, binaryAst: ByteArray) {
-        binaryAsts[srcPath] = binaryAst
-    }
-
-    fun saveFragment(srcPath: String, fragment: JsIrProgramFragment) {
-        fragments[srcPath] = fragment
-    }
-
-    abstract fun fetchArtifacts(): ModuleArtifact
-}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt

@@ -1,348 +0,0 @@
-/*
- * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
- * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
- */
-
-package org.jetbrains.kotlin.ir.backend.js.ic
-
-import org.jetbrains.kotlin.config.CompilerConfiguration
-import org.jetbrains.kotlin.ir.backend.js.*
-import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
-import org.jetbrains.kotlin.ir.declarations.IrFactory
-import org.jetbrains.kotlin.ir.declarations.IrFunction
-import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
-import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
-import org.jetbrains.kotlin.ir.util.*
-import org.jetbrains.kotlin.ir.visitors.acceptVoid
-import org.jetbrains.kotlin.js.config.JSConfigurationKeys
-import org.jetbrains.kotlin.konan.properties.propertyList
-import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
-import org.jetbrains.kotlin.library.KotlinLibrary
-import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
-import org.jetbrains.kotlin.psi2ir.descriptors.IrBuiltInsOverDescriptors
-import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
-import org.jetbrains.kotlin.storage.LockBasedStorageManager
-import java.io.File
-
-
-fun interface CacheExecutor {
-    fun execute(
-        currentModule: IrModuleFragment,
-        dependencies: Collection<IrModuleFragment>,
-        deserializer: JsIrLinker,
-        configuration: CompilerConfiguration,
-        dirtyFiles: Collection<String>?, // if null consider the whole module dirty
-        artifactCache: ArtifactCache,
-        exportedDeclarations: Set<FqName>,
-        mainArguments: List<String>?,
-    )
-}
-
-sealed class CacheUpdateStatus {
-    object FastPath : CacheUpdateStatus()
-    class NoDirtyFiles(val removed: Set<String>) : CacheUpdateStatus()
-    class Dirty(val removed: Set<String>, val updated: Set<String>, val updatedAll: Boolean) : CacheUpdateStatus()
-}
-
-class CacheUpdater(
-    private val rootModule: String,
-    private val allModules: Collection<String>,
-    private val compilerConfiguration: CompilerConfiguration,
-    private val icCachePaths: Collection<String>,
-    private val irFactory: () -> IrFactory,
-    private val mainArguments: List<String>?,
-    private val executor: CacheExecutor
-) {
-    private fun KotlinLibrary.moduleCanonicalName() = libraryFile.canonicalPath
-
-    private inner class KLibCacheUpdater(
-        private val library: KotlinLibrary,
-        private val dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
-        private val incrementalCache: IncrementalCache,
-        private val klibIncrementalCaches: Map<KotlinLibrary, IncrementalCache>
-    ) {
-        private fun invalidateCacheForModule(externalHashes: Map<IdSignature, ICHash>): Pair<Set<String>, Map<String, ICHash>> {
-            val fileFingerPrints = mutableMapOf<String, ICHash>()
-            val dirtyFiles = mutableSetOf<String>()
-
-            if (incrementalCache.klibUpdated) {
-                for ((index, file) in incrementalCache.srcFilesInOrderFromKLib.withIndex()) {
-                    // 1. get cached fingerprints
-                    val fileOldFingerprint = incrementalCache.srcFingerprints[file] ?: 0
-
-                    // 2. calculate new fingerprints
-                    val fileNewFingerprint = library.fingerprint(index)
-
-                    if (fileOldFingerprint != fileNewFingerprint) {
-                        fileFingerPrints[file] = fileNewFingerprint
-                        incrementalCache.invalidateForSrcFile(file)
-
-                        // 3. form initial dirty set
-                        dirtyFiles.add(file)
-                    }
-                }
-            }
-
-            // 4. extend dirty set with inline functions
-            do {
-                if (dirtyFiles.size == incrementalCache.srcFilesInOrderFromKLib.size) break
-
-                val oldSize = dirtyFiles.size
-                for (file in incrementalCache.srcFilesInOrderFromKLib) {
-
-                    if (file in dirtyFiles) continue
-
-                    // check for clean file
-                    val usedInlineFunctions = incrementalCache.usedFunctions[file] ?: emptyMap()
-
-                    for ((sig, oldHash) in usedInlineFunctions) {
-                        val actualHash = externalHashes[sig] ?: incrementalCache.implementedFunctions.firstNotNullOfOrNull { it[sig] }
-                        // null means inline function is from dirty file, could be a bit more optimal
-                        if (actualHash == null || oldHash != actualHash) {
-                            fileFingerPrints[file] = incrementalCache.srcFingerprints[file] ?: error("Cannot find fingerprint for $file")
-                            incrementalCache.invalidateForSrcFile(file)
-                            dirtyFiles.add(file)
-                            break
-                        }
-                    }
-                }
-            } while (oldSize != dirtyFiles.size)
-
-            // 5. invalidate file caches
-            for (deleted in incrementalCache.deletedSrcFiles) {
-                incrementalCache.invalidateForSrcFile(deleted)
-            }
-
-            return dirtyFiles to fileFingerPrints
-        }
-
-        private fun getDependencySubGraph(): Map<KotlinLibrary, List<KotlinLibrary>> {
-            val subGraph = mutableMapOf<KotlinLibrary, List<KotlinLibrary>>()
-
-            fun addDependsFor(library: KotlinLibrary) {
-                if (library in subGraph) {
-                    return
-                }
-                val dependencies = dependencyGraph[library] ?: error("Cannot find dependencies for ${library.libraryName}")
-                subGraph[library] = dependencies
-                for (dependency in dependencies) {
-                    addDependsFor(dependency)
-                }
-            }
-            addDependsFor(library)
-            return subGraph
-        }
-
-        private fun buildCacheForModule(
-            irModule: IrModuleFragment,
-            deserializer: JsIrLinker,
-            dependencies: Collection<IrModuleFragment>,
-            dirtyFiles: Collection<String>,
-            cleanInlineHashes: Map<IdSignature, ICHash>,
-            fileFingerPrints: Map<String, ICHash>
-        ) {
-            val dirtyIrFiles = irModule.files.filter { it.fileEntry.name in dirtyFiles }
-
-            val flatHashes = InlineFunctionFlatHashBuilder().apply {
-                dirtyIrFiles.forEach { it.acceptVoid(this) }
-            }.getFlatHashes()
-
-            val hashProvider = object : InlineFunctionHashProvider {
-                override fun hashForExternalFunction(declaration: IrFunction): ICHash? {
-                    return declaration.symbol.signature?.let { cleanInlineHashes[it] }
-                }
-            }
-
-            val hashBuilder = InlineFunctionHashBuilder(hashProvider, flatHashes)
-
-            val hashes = hashBuilder.buildHashes(dirtyIrFiles)
-
-            val splitPerFiles = hashes.entries.filter { !it.key.isFakeOverride && (it.key.symbol.signature?.visibleCrossFile ?: false) }
-                .groupBy({ it.key.file }) {
-                    val signature = it.key.symbol.signature ?: error("Unexpected private inline fun ${it.key.render()}")
-                    signature to it.value
-                }
-
-            val inlineGraph = hashBuilder.buildInlineGraph(hashes)
-
-            dirtyIrFiles.forEach { irFile ->
-                val fileName = irFile.fileEntry.name
-                incrementalCache.updateHashes(
-                    srcPath = fileName,
-                    fingerprint = fileFingerPrints[fileName] ?: error("No fingerprint found for file $fileName"),
-                    usedFunctions = inlineGraph[irFile],
-                    implementedFunctions = splitPerFiles[irFile]?.toMap()
-                )
-            }
-
-            // TODO: actual way of building a cache could change in future
-            executor.execute(
-                irModule, dependencies, deserializer, compilerConfiguration, dirtyFiles, incrementalCache, emptySet(), mainArguments
-            )
-        }
-
-        fun checkLibrariesHash(): Boolean {
-            val flatHash = File(library.moduleCanonicalName()).fileHashForIC()
-            val dependencies = dependencyGraph[library] ?: error("Cannot find dependencies for ${library.libraryName}")
-
-            var transHash = flatHash
-            for (dep in dependencies) {
-                val depCache = klibIncrementalCaches[dep] ?: error("Cannot cache info for ${dep.libraryName}")
-                transHash = transHash.combineWith(depCache.klibTransitiveHash)
-            }
-            return incrementalCache.checkAndUpdateCacheFastInfo(flatHash, transHash)
-        }
-
-        fun actualizeCacheForModule(): CacheUpdateStatus {
-            // 1. Invalidate
-            val dependencies = dependencyGraph[library]!!
-
-            val incrementalCache = klibIncrementalCaches[library] ?: error("No cache provider for $library")
-
-            val sigHashes = mutableMapOf<IdSignature, ICHash>()
-            dependencies.forEach { lib ->
-                klibIncrementalCaches[lib]?.let { libCache ->
-                    libCache.fetchCacheDataForDependency()
-                    libCache.implementedFunctions.forEach { sigHashes.putAll(it) }
-                }
-            }
-
-            incrementalCache.fetchFullCacheData()
-            val (dirtySet, fileFingerPrints) = invalidateCacheForModule(sigHashes)
-            val removed = incrementalCache.deletedSrcFiles
-
-            if (dirtySet.isEmpty()) {
-                // up-to-date
-                incrementalCache.commitCacheForRemovedSrcFiles()
-                return CacheUpdateStatus.NoDirtyFiles(removed)
-            }
-
-            // 2. Build
-            val jsIrLinkerProcessor = JsIrLinkerLoader(compilerConfiguration, library, getDependencySubGraph(), irFactory())
-            val (jsIrLinker, currentIrModule, irModules) = jsIrLinkerProcessor.processJsIrLinker(dirtySet)
-
-            val currentModuleDeserializer = jsIrLinker.moduleDeserializer(currentIrModule.descriptor)
-
-            incrementalCache.implementedFunctions.forEach { sigHashes.putAll(it) }
-
-            for (dirtySrcFile in dirtySet) {
-                val signatureMapping = currentModuleDeserializer.signatureDeserializerForFile(dirtySrcFile).signatureToIndexMapping()
-                incrementalCache.updateSignatureToIdMapping(dirtySrcFile, signatureMapping)
-            }
-
-            buildCacheForModule(currentIrModule, jsIrLinker, irModules, dirtySet, sigHashes, fileFingerPrints)
-
-            val updatedAll = dirtySet.size == incrementalCache.srcFilesInOrderFromKLib.size
-            incrementalCache.commitCacheForRebuiltSrcFiles(currentIrModule.name.asString())
-            // invalidated and re-built
-            return CacheUpdateStatus.Dirty(removed, dirtySet, updatedAll)
-        }
-    }
-
-    private fun loadLibraries(): Map<String, KotlinLibrary> {
-        val allResolvedDependencies = jsResolveLibraries(
-            allModules,
-            compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList(),
-            compilerConfiguration[IrMessageLogger.IR_MESSAGE_LOGGER].toResolverLogger()
-        )
-
-        return allResolvedDependencies.getFullList().associateBy { it.moduleCanonicalName() }
-    }
-
-    private fun buildDependenciesGraph(libraries: Map<String, KotlinLibrary>): Map<KotlinLibrary, List<KotlinLibrary>> {
-        val nameToKotlinLibrary: Map<String, KotlinLibrary> = libraries.values.associateBy { it.moduleName }
-        return libraries.values.associateWith {
-            it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
-                nameToKotlinLibrary[depName] ?: error("No Library found for $depName")
-            }
-        }
-    }
-
-    fun actualizeCaches(callback: (CacheUpdateStatus, String) -> Unit): List<ModuleArtifact> {
-        val libraries = loadLibraries()
-        val dependencyGraph = buildDependenciesGraph(libraries)
-        val configHash = compilerConfiguration.configHashForIC()
-
-        val cacheMap = libraries.values.zip(icCachePaths).toMap()
-
-        val klibIncrementalCaches = mutableMapOf<KotlinLibrary, IncrementalCache>()
-
-        val visitedLibraries = mutableSetOf<KotlinLibrary>()
-        fun visitDependency(library: KotlinLibrary) {
-            if (library in visitedLibraries) return
-            visitedLibraries.add(library)
-
-            val libraryDeps = dependencyGraph[library] ?: error("Unknown library ${library.libraryName}")
-            libraryDeps.forEach { visitDependency(it) }
-
-            val cachePath = cacheMap[library] ?: error("Unknown cache for library ${library.libraryName}")
-            val incrementalCache = IncrementalCache(library, cachePath)
-            klibIncrementalCaches[library] = incrementalCache
-
-            incrementalCache.invalidateCacheForNewConfig(configHash)
-            val cacheUpdater = KLibCacheUpdater(library, dependencyGraph, incrementalCache, klibIncrementalCaches)
-            val updateStatus = when {
-                cacheUpdater.checkLibrariesHash() -> CacheUpdateStatus.FastPath
-                else -> cacheUpdater.actualizeCacheForModule()
-            }
-            callback(updateStatus, library.libraryFile.path)
-        }
-
-        val rootModuleCanonical = File(rootModule).canonicalPath
-        val mainLibrary = libraries[rootModuleCanonical] ?: error("Main library not found in libraries: $rootModuleCanonical")
-        visitDependency(mainLibrary)
-        return klibIncrementalCaches.map { it.value.fetchArtifacts() }
-    }
-}
-
-
-// Used for tests only
-fun rebuildCacheForDirtyFiles(
-    library: KotlinLibrary,
-    configuration: CompilerConfiguration,
-    dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
-    dirtyFiles: Collection<String>?,
-    artifactCache: ArtifactCache,
-    irFactory: IrFactory,
-    exportedDeclarations: Set<FqName>,
-    mainArguments: List<String>?,
-): String {
-    val jsIrLinkerProcessor = JsIrLinkerLoader(configuration, library, dependencyGraph, irFactory)
-    val (jsIrLinker, currentIrModule, irModules) = jsIrLinkerProcessor.processJsIrLinker(dirtyFiles)
-
-    buildCacheForModuleFiles(
-        currentIrModule,
-        irModules,
-        jsIrLinker,
-        configuration,
-        dirtyFiles,
-        artifactCache,
-        exportedDeclarations,
-        mainArguments
-    )
-    return currentIrModule.name.asString()
-}
-
-@Suppress("UNUSED_PARAMETER")
-fun buildCacheForModuleFiles(
-    currentModule: IrModuleFragment,
-    dependencies: Collection<IrModuleFragment>,
-    deserializer: JsIrLinker,
-    configuration: CompilerConfiguration,
-    dirtyFiles: Collection<String>?, // if null consider the whole module dirty
-    artifactCache: ArtifactCache,
-    exportedDeclarations: Set<FqName>,
-    mainArguments: List<String>?,
-) {
-    compileWithIC(
-        currentModule,
-        configuration = configuration,
-        deserializer = deserializer,
-        dependencies = dependencies,
-        mainArguments = mainArguments,
-        exportedDeclarations = exportedDeclarations,
-        filesToLower = dirtyFiles?.toSet(),
-        artifactCache = artifactCache,
-    )
-}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/JsPropertyAccessorInlineLowering.kt

@@ -21,10 +21,11 @@ class JsPropertyAccessorInlineLowering(
             return false
 
         // Member properties could be safely inlined, because initialization processed via parent declaration
-        if (!isTopLevel) return true
+        if (!isTopLevel && !context.icCompatibleIr2Js.incrementalCacheEnabled)
+            return true
 
         // TODO: teach the deserializer to load constant property initializers
-        if (context.icCompatibleIr2Js) {
+        if (context.icCompatibleIr2Js.isCompatible) {
             val accessFile = accessContainer.fileOrNull ?: return false
             val file = fileOrNull ?: return false
 
@@ -49,4 +50,4 @@ class JsPropertyAccessorInlineLowering(
                 false
         }
     }
-}
+}

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/transformers/irToJs/IrModuleToJsTransformerTmp.kt

@@ -62,6 +62,8 @@ enum class TranslationMode(
     }
 }
 
+class JsIrFragmentAndBinaryAst(val irFile: IrFile, val fragment: JsIrProgramFragment, val binaryAst: ByteArray)
+
 class IrModuleToJsTransformerTmp(
     private val backendContext: JsIrBackendContext,
     private val mainArguments: List<String>?,
@@ -125,14 +127,10 @@ class IrModuleToJsTransformerTmp(
         return CompilerResult(result, dts)
     }
 
-    class SrcFileFragmentAndBinaryAst(val srcPath: String, val fragment: JsIrProgramFragment, val binaryAst: ByteArray)
-
-    fun generateBinaryAst(files: Iterable<IrFile>, allModules: Iterable<IrModuleFragment>): List<SrcFileFragmentAndBinaryAst> {
+    fun generateBinaryAst(files: Collection<IrFile>, allModules: Collection<IrModuleFragment>): List<JsIrFragmentAndBinaryAst> {
         val exportModelGenerator = ExportModelGenerator(backendContext, generateNamespacesForPackages = true)
 
-        val exportData = files.associate { file ->
-            file to exportModelGenerator.generateExportWithExternals(file)
-        }
+        val exportData = files.map { it to exportModelGenerator.generateExportWithExternals(it) }
 
         allModules.forEach {
             it.files.forEach {
@@ -141,13 +139,12 @@ class IrModuleToJsTransformerTmp(
         }
 
         val serializer = JsIrAstSerializer()
-        return files.map { f ->
-            val exports = exportData[f]!! // TODO
-            val fragment = generateProgramFragment(f, exports, minimizedMemberNames = false)
+        return exportData.map { (file, exports) ->
+            val fragment = generateProgramFragment(file, exports, minimizedMemberNames = false)
             val output = ByteArrayOutputStream()
             serializer.serialize(fragment, output)
             val binaryAst = output.toByteArray()
-            SrcFileFragmentAndBinaryAst(f.fileEntry.name, fragment, binaryAst)
+            JsIrFragmentAndBinaryAst(file, fragment, binaryAst)
         }
     }
 

compiler/tests-compiler-utils/tests/org/jetbrains/kotlin/codegen/TestModel.kt

@@ -5,6 +5,7 @@
 
 package org.jetbrains.kotlin.codegen
 
+import org.jetbrains.kotlin.ir.backend.js.ic.DirtyFileState
 import java.io.File
 import java.util.regex.Pattern
 
@@ -43,19 +44,13 @@ class ModuleInfo(val moduleName: String) {
     class ModuleStep(
         val id: Int,
         val dependencies: Collection<String>,
-        val dirtyFiles: Collection<String>,
         val modifications: List<Modification>,
-        val directives: Set<StepDirectives>
+        val expectedFileStats: Map<String, Set<String>>
     )
 
     val steps = mutableListOf<ModuleStep>()
 }
 
-enum class StepDirectives(val mnemonic: String) {
-    FAST_PATH_UPDATE("FP"),
-    UNUSED_MODULE("UNUSED")
-}
-
 const val MODULES_LIST = "MODULES"
 const val PROJECT_INFO_FILE = "project.info"
 const val MODULE_INFO_FILE = "module.info"
@@ -203,18 +198,10 @@ class ModuleInfoParser(infoFile: File) : InfoParser<ModuleInfo>(infoFile) {
         return modifications
     }
 
-    private fun MutableSet<StepDirectives>.parseStepFlags(flags: String) {
-        val tokens = flags.trim().split(" ").map { it.trim() }
-        for (directive in StepDirectives.values()) {
-            if (directive.mnemonic in tokens) add(directive)
-        }
-    }
-
     private fun parseSteps(firstId: Int, lastId: Int): List<ModuleInfo.ModuleStep> {
+        val expectedFileStats = mutableMapOf<String, Set<String>>()
         val dependencies = mutableSetOf<String>()
-        val dirtyFiles = mutableSetOf<String>()
         val modifications = mutableListOf<ModuleInfo.Modification>()
-        val directives = mutableSetOf<StepDirectives>()
 
         loop { line ->
             if (line.matches(STEP_PATTERN.toRegex()))
@@ -225,17 +212,30 @@ class ModuleInfoParser(infoFile: File) : InfoParser<ModuleInfo>(infoFile) {
             if (opIndex < 0) throwSyntaxError(line)
             val op = line.substring(0, opIndex)
 
-            when (op) {
-                "dependencies" -> line.substring(opIndex + 1).split(",").filter { it.isNotBlank() }.forEach { dependencies.add(it.trim()) }
-                "dirty" -> line.substring(opIndex + 1).split(",").filter { it.isNotBlank() }.forEach { dirtyFiles.add(it.trim()) }
-                "modifications" -> modifications.addAll(parseModifications())
-                "flags" -> directives.parseStepFlags(line.substring(opIndex + 1))
-                else -> println(diagnosticMessage("Unknown op $op", line))
+            fun getOpArgs() = line.substring(opIndex + 1).splitAndTrim()
+
+            val expectedState = DirtyFileState.values().find { it.str == op }
+            if (expectedState != null) {
+                expectedFileStats[expectedState.str] = getOpArgs().toSet()
+            } else {
+                when (op) {
+                    "dependencies" -> getOpArgs().forEach { dependencies.add(it) }
+                    "modifications" -> modifications.addAll(parseModifications())
+                    else -> println(diagnosticMessage("Unknown op $op", line))
+                }
             }
+
             false
         }
 
-        return (firstId..lastId).map { ModuleInfo.ModuleStep(it, dependencies, dirtyFiles, modifications, directives) }
+        return (firstId..lastId).map {
+            ModuleInfo.ModuleStep(
+                id = it,
+                dependencies = dependencies,
+                modifications = modifications,
+                expectedFileStats = expectedFileStats
+            )
+        }
     }
 
     override fun parse(entryName: String): ModuleInfo {