[JS IR] Use topological order for loading klibs in JS IR IC

Generally, the library order doesn't matter.

 However, after lowering we need to post process
 lowered klibs in topological order:
  - commit incremental cache artifacts,
  - produce JS AST.

 The patch uses an ability of the Kotlin library resolver to load klibs
 in topological order and drops its reimplementation from IC infrastructure.

 There is one side effect:
   klibs which are not reachable from the main module, will produce their JS.
   This should be more correct than just ignoring them.

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -659,12 +659,12 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
 
             val artifacts = cacheUpdater.actualizeCaches()
             messageCollector.report(INFO, "IC rebuilt overall time: ${System.currentTimeMillis() - start}ms")
-            for ((event, duration) in cacheUpdater.getStopwatchLaps()) {
+            for ((event, duration) in cacheUpdater.getStopwatchLastLaps()) {
                 messageCollector.report(INFO, "  $event: ${(duration / 1e6).toInt()}ms")
             }
 
             var libIndex = 0
-            for ((libFile, srcFiles) in cacheUpdater.getDirtyFileStats()) {
+            for ((libFile, srcFiles) in cacheUpdater.getDirtyFileLastStats()) {
                 val (msg, showFiles) = when {
                     srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) } -> "fully rebuilt due to clean build" to false
                     srcFiles.values.all { it.contains(DirtyFileState.MODIFIED_CONFIG) } -> "fully rebuilt due to config modification" to false

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -15,9 +15,9 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys
 import org.jetbrains.kotlin.konan.properties.propertyList
 import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS
 import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder
 import org.jetbrains.kotlin.name.FqName
 import java.io.File
-import java.security.MessageDigest
 import java.util.EnumSet
 
 fun interface JsIrCompilerICInterface {
@@ -48,31 +48,24 @@ enum class DirtyFileState(val str: String) {
 
 class CacheUpdater(
     mainModule: String,
-    allModules: Collection<String>,
+    private val allModules: Collection<String>,
     private val icCacheRootDir: String,
     private val compilerConfiguration: CompilerConfiguration,
     private val irFactory: () -> IrFactory,
     private val mainArguments: List<String>?,
     private val compilerInterfaceFactory: JsIrCompilerICInterfaceFactory
 ) {
-    private val stopwatch = StopwatchIC().apply { startNext("Loading klibs") }
+    private val stopwatch = StopwatchIC()
-
-    private val libraries = loadLibraries(allModules)
-    private val dependencyGraph = buildDependenciesGraph(libraries)
-    private val configHash = compilerConfiguration.configHashForIC()
-
-    private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)
-    private val mainLibrary = libraries[mainLibraryFile] ?: notFoundIcError("main library", mainLibraryFile)
 
     private val dirtyFileStats = KotlinSourceFileMutableMap<EnumSet<DirtyFileState>>()
 
-    init {
+    private val configHash = compilerConfiguration.configHashForIC()
-        stopwatch.stop()
-    }
 
-    fun getDirtyFileStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats
+    private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)
 
-    fun getStopwatchLaps() = stopwatch.laps
+    fun getDirtyFileLastStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats
+
+    fun getStopwatchLastLaps() = stopwatch.laps
 
     private fun MutableMap<KotlinSourceFile, EnumSet<DirtyFileState>>.addDirtFileStat(srcFile: KotlinSourceFile, state: DirtyFileState) {
         when (val stats = this[srcFile]) {
@@ -81,44 +74,41 @@ class CacheUpdater(
         }
     }
 
-    private fun loadLibraries(allModules: Collection<String>): Map<KotlinLibraryFile, KotlinLibrary> {
-        val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList()
-        val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger)
-        return allResolvedDependencies.getFullList().associateBy { KotlinLibraryFile(it) }
-    }
-
-    private fun buildDependenciesGraph(libraries: Map<KotlinLibraryFile, KotlinLibrary>): Map<KotlinLibrary, List<KotlinLibrary>> {
-        val nameToKotlinLibrary = libraries.values.associateBy { it.moduleName }
-        return libraries.values.associateWith {
-            it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
-                nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName")
-            }
-        }
-    }
-
-    private fun <T> mapLibrariesInOrder(f: (KotlinLibraryFile) -> T): List<T> {
-        val visited = HashSet<KotlinLibrary>(libraries.size)
-        val result = ArrayList<T>(libraries.size)
-        fun visitLibrary(lib: KotlinLibrary) {
-            if (visited.add(lib)) {
-                dependencyGraph[lib]?.forEach(::visitLibrary)
-                result += f(KotlinLibraryFile(lib))
-            }
-        }
-
-        visitLibrary(mainLibrary)
-        return result
-    }
-
     private inner class CacheUpdaterInternal {
         val signatureHashCalculator = IdSignatureHashCalculator()
 
-        private val incrementalCaches = run {
+        // libraries in topological order: [stdlib, ..., main]
-            val md5 = MessageDigest.getInstance("MD5")
+        val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") {
-            libraries.entries.associate { (libFile, lib) ->
+            val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList()
+            val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger)
+
+            val libraries = allResolvedDependencies.getFullList(TopologicalLibraryOrder).let { resolvedLibraries ->
+                val mainLibraryIndex = resolvedLibraries.indexOfLast {
+                    KotlinLibraryFile(it) == mainLibraryFile
+                }.takeIf { it >= 0 } ?: notFoundIcError("main library", mainLibraryFile)
+
+                when (mainLibraryIndex) {
+                    resolvedLibraries.lastIndex -> resolvedLibraries
+                    else -> resolvedLibraries.filterIndexedTo(ArrayList(resolvedLibraries.size)) { index, _ ->
+                        index != mainLibraryIndex
+                    }.apply { add(resolvedLibraries[mainLibraryIndex]) }
+                }
+            }
+
+            val nameToKotlinLibrary = libraries.associateBy { it.moduleName }
+
+            libraries.associateWith {
+                it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
+                    nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName")
+                }
+            }
+        }
+
+        private val incrementalCaches = stopwatch.measure("Incremental cache - initializing") {
+            libraryDependencies.keys.associate { lib ->
+                val libFile = KotlinLibraryFile(lib)
                 val file = File(libFile.path)
-                val hash = md5.digest(file.absolutePath.encodeToByteArray()).joinToString("", transform = { "%02x".format(it) })
+                libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.absolutePath.stringHashForIC()}.${file.name}"))
-                libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.name}.$hash"))
             }
         }
 
@@ -511,7 +501,8 @@ class CacheUpdater(
             jsIrLinker: JsIrLinker,
             loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>
         ): Map<KotlinLibraryFile, IncrementalCacheArtifact> {
-            return mapLibrariesInOrder { libFile ->
+            return libraryDependencies.keys.associate { library ->
+                val libFile = KotlinLibraryFile(library)
                 val incrementalCache = getLibIncrementalCache(libFile)
                 val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile)
                 val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor)
@@ -520,7 +511,7 @@ class CacheUpdater(
                     KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping()
                 }
                 libFile to incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping)
-            }.toMap()
+            }
         }
     }
 
@@ -528,50 +519,40 @@ class CacheUpdater(
         incrementalCacheArtifacts: Map<KotlinLibraryFile, IncrementalCacheArtifact>,
         moduleNames: Map<KotlinLibraryFile, String>,
         rebuiltFileFragments: KotlinSourceFileMap<JsIrProgramFragment>
-    ): List<ModuleArtifact> {
+    ): List<ModuleArtifact> = stopwatch.measure("Incremental cache - committing artifacts") {
-        stopwatch.startNext("Incremental cache - committing artifacts")
+        incrementalCacheArtifacts.map { (libFile, incrementalCacheArtifact) ->
-
-        val artifacts = mapLibrariesInOrder { libFile ->
-            val incrementalCacheArtifact = incrementalCacheArtifacts[libFile] ?: notFoundIcError("cache artifact", libFile)
-            val library = libraries[libFile] ?: notFoundIcError("library", libFile)
             incrementalCacheArtifact.buildModuleArtifactAndCommitCache(
                 moduleName = moduleNames[libFile] ?: notFoundIcError("module name", libFile),
-                externalModuleName = library.jsOutputName,
                 rebuiltFileFragments = rebuiltFileFragments[libFile] ?: emptyMap()
             )
         }
-
-        stopwatch.stop()
-        return artifacts
     }
 
     private fun compileDirtyFiles(
         compilerForIC: JsIrCompilerICInterface,
         loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>,
         dirtyFiles: Map<KotlinLibraryFile, Set<KotlinSourceFile>>
-    ): MutableList<Triple<KotlinLibraryFile, KotlinSourceFile, () -> JsIrProgramFragment>> {
+    ): MutableList<Triple<KotlinLibraryFile, KotlinSourceFile, () -> JsIrProgramFragment>> =
-        stopwatch.startNext("Processing IR - lowering")
+        stopwatch.measure("Processing IR - lowering") {
-
+            val dirtyFilesForCompiling = mutableListOf<IrFile>()
-        val dirtyFilesForCompiling = mutableListOf<IrFile>()
+            val dirtyFilesForRestoring = mutableListOf<Pair<KotlinLibraryFile, KotlinSourceFile>>()
-        val dirtyFilesForRestoring = mutableListOf<Pair<KotlinLibraryFile, KotlinSourceFile>>()
+            for ((libFile, libFragment) in loadedFragments) {
-        for ((libFile, libFragment) in loadedFragments) {
+                val dirtySrcFiles = dirtyFiles[libFile] ?: continue
-            val dirtySrcFiles = dirtyFiles[libFile] ?: continue
+                for (irFile in libFragment.files) {
-            for (irFile in libFragment.files) {
+                    val srcFile = KotlinSourceFile(irFile)
-                val srcFile = KotlinSourceFile(irFile)
+                    if (srcFile in dirtySrcFiles) {
-                if (srcFile in dirtySrcFiles) {
+                        dirtyFilesForCompiling += irFile
-                    dirtyFilesForCompiling += irFile
+                        dirtyFilesForRestoring += libFile to srcFile
-                    dirtyFilesForRestoring += libFile to srcFile
+                    }
                 }
             }
-        }
 
-        val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments)
+            val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments)
 
-        stopwatch.stop()
+            dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile ->
-        return dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile ->
+                Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i])
-            Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i])
+            }
         }
-    }
 
     private data class IrForDirtyFilesAndCompiler(
         val incrementalCacheArtifacts: Map<KotlinLibraryFile, IncrementalCacheArtifact>,
@@ -581,9 +562,6 @@ class CacheUpdater(
     )
 
     private fun loadIrForDirtyFilesAndInitCompiler(): IrForDirtyFilesAndCompiler {
-        dirtyFileStats.clear()
-
-        stopwatch.startNext("Incremental cache - initializing")
         val updater = CacheUpdaterInternal()
 
         stopwatch.startNext("Modified files - checking hashes and collecting")
@@ -593,7 +571,7 @@ class CacheUpdater(
         val dirtyFileExports = updater.collectExportedSymbolsForDirtyFiles(modifiedFiles)
 
         stopwatch.startNext("Modified files - loading and linking IR")
-        val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, mainLibrary, dependencyGraph, irFactory())
+        val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, updater.libraryDependencies, irFactory())
         var loadedIr = jsIrLinkerLoader.loadIr(dirtyFileExports)
 
         var iterations = 0
@@ -630,8 +608,8 @@ class CacheUpdater(
         }
 
         stopwatch.startNext("Processing IR - initializing backend context")
-        val mainModule = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main lib loaded fragment", mainLibraryFile)
+        val mainModuleFragment = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main module fragment", mainLibraryFile)
-        val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModule, compilerConfiguration)
+        val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModuleFragment, compilerConfiguration)
 
         // Load declarations referenced during `context` initialization
         loadedIr.linker.loadUnboundSymbols(true)
@@ -639,7 +617,7 @@ class CacheUpdater(
         val dirtyFiles = dirtyFileExports.entries.associateTo(HashMap(dirtyFileExports.size)) { it.key to HashSet(it.value.keys) }
 
         stopwatch.startNext("Processing IR - updating intrinsics and builtins dependencies")
-        updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModule, loadedIr.loadedFragments, dirtyFiles)
+        updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModuleFragment, loadedIr.loadedFragments, dirtyFiles)
 
         stopwatch.startNext("Incremental cache - building artifacts")
         val incrementalCachesArtifacts = updater.buildCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments)
@@ -666,18 +644,19 @@ class CacheUpdater(
 
     private fun generateIrFragments(
         generators: MutableList<Triple<KotlinLibraryFile, KotlinSourceFile, () -> JsIrProgramFragment>>
-    ): KotlinSourceFileMap<JsIrProgramFragment> {
+    ): KotlinSourceFileMap<JsIrProgramFragment> = stopwatch.measure("Processing IR - generating program fragments") {
-        stopwatch.startNext("Processing IR - generating program fragments")
         val rebuiltFragments = KotlinSourceFileMutableMap<JsIrProgramFragment>()
         while (generators.isNotEmpty()) {
             val (libFile, srcFile, fragmentGenerator) = generators.removeFirst()
             rebuiltFragments[libFile, srcFile] = fragmentGenerator()
         }
-        stopwatch.stop()
+        rebuiltFragments
-        return rebuiltFragments
     }
 
     fun actualizeCaches(): List<ModuleArtifact> {
+        stopwatch.clear()
+        dirtyFileStats.clear()
+
         val (incrementalCachesArtifacts, moduleNames, generators) = loadIrAndMakeIrFragmentGenerators()
 
         val rebuiltFragments = generateIrFragments(generators)
@@ -704,7 +683,7 @@ fun rebuildCacheForDirtyFiles(
     val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLibraryHeader(library).sourceFiles
     val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })
 
-    val jsIrLoader = JsIrLinkerLoader(configuration, library, dependencyGraph, irFactory)
+    val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, irFactory)
     val (jsIrLinker, irModules) = jsIrLoader.loadIr(KotlinSourceFileMap<KotlinSourceFileExports>(modifiedFiles), true)
 
     val currentIrModule = irModules[libFile] ?: notFoundIcError("loaded fragment", libFile)

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt

@@ -69,6 +69,12 @@ internal class StopwatchIC {
     val laps: List<Pair<String, Long>>
         get() = lapsImpl
 
+    fun clear() {
+        lapStart = 0
+        lapDescription = null
+        lapsImpl.clear()
+    }
+
     fun startNext(description: String) {
         val now = System.nanoTime()
         stop(now)
@@ -82,4 +88,11 @@ internal class StopwatchIC {
         }
         lapDescription = null
     }
+
+    inline fun <T> measure(description: String, f: () -> T): T {
+        startNext(description)
+        val result = f()
+        stop()
+        return result
+    }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.ir.backend.js.ic
 
 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.ir.backend.js.jsOutputName
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.impl.javaFile
@@ -85,7 +86,7 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val
         val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile ->
             commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
         }
-        return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts)
+        return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts, library.jsOutputName)
     }
 
     data class ModifiedFiles(

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt

@@ -54,11 +54,11 @@ internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val
 internal class IncrementalCacheArtifact(
     private val artifactsDir: File,
     private val forceRebuildJs: Boolean,
-    private val srcCacheActions: List<SourceFileCacheArtifact>
+    private val srcCacheActions: List<SourceFileCacheArtifact>,
+    private val externalModuleName: String?
 ) {
     fun buildModuleArtifactAndCommitCache(
         moduleName: String,
-        externalModuleName: String?,
         rebuiltFileFragments: Map<KotlinSourceFile, JsIrProgramFragment>,
     ): ModuleArtifact {
         val serializer = JsIrAstSerializer()

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt

@@ -39,7 +39,6 @@ internal fun JsIrLinker.loadUnboundSymbols(checkNoUnbound: Boolean) {
 
 internal class JsIrLinkerLoader(
     private val compilerConfiguration: CompilerConfiguration,
-    private val library: KotlinLibrary,
     private val dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
     private val irFactory: IrFactory
 ) {
@@ -87,11 +86,12 @@ internal class JsIrLinkerLoader(
         val loadedModules = loadModules()
         val jsIrLinker = createLinker(loadedModules)
 
+        val mainLibrary = dependencyGraph.keys.lastOrNull() ?: notFoundIcError("main library")
         val irModules = loadedModules.entries.associate { (descriptor, module) ->
             val libraryFile = KotlinLibraryFile(module)
             val modifiedStrategy = when {
                 loadAllIr -> DeserializationStrategy.ALL
-                module == library -> DeserializationStrategy.ALL
+                module == mainLibrary -> DeserializationStrategy.ALL
                 else -> DeserializationStrategy.EXPLICITLY_EXPORTED
             }
             val modified = modifiedFiles[libraryFile] ?: emptyMap()

js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt

@@ -44,7 +44,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
     companion object {
         private val TEST_DATA_DIR_PATH = System.getProperty("kotlin.js.test.root.out.dir") ?: error("'kotlin.js.test.root.out.dir' is not set")
         private const val BOX_FUNCTION_NAME = "box"
-        private const val STDLIB_ALIAS = "stdlib"
 
         private const val STDLIB_MODULE_NAME = "kotlin-kotlin-stdlib-js-ir"
         private val STDLIB_KLIB = File(System.getProperty("kotlin.js.stdlib.klib.path") ?: error("Please set stdlib path")).canonicalPath
@@ -240,7 +239,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
                 )
 
                 val icCaches = cacheUpdater.actualizeCaches()
-                verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileStats(), testInfo)
+                verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileLastStats(), testInfo)
 
                 val mainModuleName = icCaches.last().moduleExternalName
                 val jsExecutableProducer = JsExecutableProducer(

js/js.translator/testData/incremental/invalidation/removeModule/project.info

@@ -5,4 +5,4 @@ STEP 0:
     dirty js: lib1, lib2, main
 STEP 1:
     libs: lib1, lib2, main
-    dirty js: lib2
+    dirty js: lib1, lib2