From 8cde0a81bc5fd7b6fd61b508166733c17c6fd2af Mon Sep 17 00:00:00 2001 From: Alexander Korepanov Date: Wed, 16 Nov 2022 11:47:24 +0100 Subject: [PATCH] [JS IR] Use topological order for loading klibs in JS IR IC Generally, the library order doesn't matter. However, after lowering we need to post process lowered klibs in topological order: - commit incremental cache artifacts, - produce JS AST. The patch uses an ability of the Kotlin library resolver to load klibs in topological order and drops its reimplementation from IC infrastructure. There is one side effect: klibs which are not reachable from the main module, will produce their JS. This should be more correct than just ignoring them. --- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 4 +- .../kotlin/ir/backend/js/ic/CacheUpdater.kt | 161 ++++++++---------- .../kotlin/ir/backend/js/ic/ICUtils.kt | 13 ++ .../ir/backend/js/ic/IncrementalCache.kt | 3 +- .../backend/js/ic/IncrementalCacheArtifact.kt | 4 +- .../ir/backend/js/ic/JsIrLinkerLoader.kt | 4 +- .../incremental/AbstractInvalidationTest.kt | 3 +- .../invalidation/removeModule/project.info | 2 +- 8 files changed, 93 insertions(+), 101 deletions(-) diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index ad3a44c1764..cf149dcf5cc 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -659,12 +659,12 @@ class K2JsIrCompiler : CLICompiler() { val artifacts = cacheUpdater.actualizeCaches() messageCollector.report(INFO, "IC rebuilt overall time: ${System.currentTimeMillis() - start}ms") - for ((event, duration) in cacheUpdater.getStopwatchLaps()) { + for ((event, duration) in cacheUpdater.getStopwatchLastLaps()) { messageCollector.report(INFO, " $event: ${(duration / 1e6).toInt()}ms") } var libIndex = 0 - for ((libFile, srcFiles) in cacheUpdater.getDirtyFileStats()) { + for ((libFile, srcFiles) in cacheUpdater.getDirtyFileLastStats()) { val (msg, showFiles) = when { srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) } -> "fully rebuilt due to clean build" to false srcFiles.values.all { it.contains(DirtyFileState.MODIFIED_CONFIG) } -> "fully rebuilt due to config modification" to false diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index a9dd86ce071..cd7a2343261 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -15,9 +15,9 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.konan.properties.propertyList import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.name.FqName import java.io.File -import java.security.MessageDigest import java.util.EnumSet fun interface JsIrCompilerICInterface { @@ -48,31 +48,24 @@ enum class DirtyFileState(val str: String) { class CacheUpdater( mainModule: String, - allModules: Collection, + private val allModules: Collection, private val icCacheRootDir: String, private val compilerConfiguration: CompilerConfiguration, private val irFactory: () -> IrFactory, private val mainArguments: List?, private val compilerInterfaceFactory: JsIrCompilerICInterfaceFactory ) { - private val stopwatch = StopwatchIC().apply { startNext("Loading klibs") } - - private val libraries = loadLibraries(allModules) - private val dependencyGraph = buildDependenciesGraph(libraries) - private val configHash = compilerConfiguration.configHashForIC() - - private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) - private val mainLibrary = libraries[mainLibraryFile] ?: notFoundIcError("main library", mainLibraryFile) + private val stopwatch = StopwatchIC() private val dirtyFileStats = KotlinSourceFileMutableMap>() - init { - stopwatch.stop() - } + private val configHash = compilerConfiguration.configHashForIC() - fun getDirtyFileStats(): KotlinSourceFileMap> = dirtyFileStats + private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) - fun getStopwatchLaps() = stopwatch.laps + fun getDirtyFileLastStats(): KotlinSourceFileMap> = dirtyFileStats + + fun getStopwatchLastLaps() = stopwatch.laps private fun MutableMap>.addDirtFileStat(srcFile: KotlinSourceFile, state: DirtyFileState) { when (val stats = this[srcFile]) { @@ -81,44 +74,41 @@ class CacheUpdater( } } - private fun loadLibraries(allModules: Collection): Map { - val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList() - val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger) - return allResolvedDependencies.getFullList().associateBy { KotlinLibraryFile(it) } - } - - private fun buildDependenciesGraph(libraries: Map): Map> { - val nameToKotlinLibrary = libraries.values.associateBy { it.moduleName } - return libraries.values.associateWith { - it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName -> - nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName") - } - } - } - - private fun mapLibrariesInOrder(f: (KotlinLibraryFile) -> T): List { - val visited = HashSet(libraries.size) - val result = ArrayList(libraries.size) - fun visitLibrary(lib: KotlinLibrary) { - if (visited.add(lib)) { - dependencyGraph[lib]?.forEach(::visitLibrary) - result += f(KotlinLibraryFile(lib)) - } - } - - visitLibrary(mainLibrary) - return result - } - private inner class CacheUpdaterInternal { val signatureHashCalculator = IdSignatureHashCalculator() - private val incrementalCaches = run { - val md5 = MessageDigest.getInstance("MD5") - libraries.entries.associate { (libFile, lib) -> + // libraries in topological order: [stdlib, ..., main] + val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") { + val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList() + val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger) + + val libraries = allResolvedDependencies.getFullList(TopologicalLibraryOrder).let { resolvedLibraries -> + val mainLibraryIndex = resolvedLibraries.indexOfLast { + KotlinLibraryFile(it) == mainLibraryFile + }.takeIf { it >= 0 } ?: notFoundIcError("main library", mainLibraryFile) + + when (mainLibraryIndex) { + resolvedLibraries.lastIndex -> resolvedLibraries + else -> resolvedLibraries.filterIndexedTo(ArrayList(resolvedLibraries.size)) { index, _ -> + index != mainLibraryIndex + }.apply { add(resolvedLibraries[mainLibraryIndex]) } + } + } + + val nameToKotlinLibrary = libraries.associateBy { it.moduleName } + + libraries.associateWith { + it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName -> + nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName") + } + } + } + + private val incrementalCaches = stopwatch.measure("Incremental cache - initializing") { + libraryDependencies.keys.associate { lib -> + val libFile = KotlinLibraryFile(lib) val file = File(libFile.path) - val hash = md5.digest(file.absolutePath.encodeToByteArray()).joinToString("", transform = { "%02x".format(it) }) - libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.name}.$hash")) + libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.absolutePath.stringHashForIC()}.${file.name}")) } } @@ -511,7 +501,8 @@ class CacheUpdater( jsIrLinker: JsIrLinker, loadedFragments: Map ): Map { - return mapLibrariesInOrder { libFile -> + return libraryDependencies.keys.associate { library -> + val libFile = KotlinLibraryFile(library) val incrementalCache = getLibIncrementalCache(libFile) val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile) val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor) @@ -520,7 +511,7 @@ class CacheUpdater( KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() } libFile to incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping) - }.toMap() + } } } @@ -528,50 +519,40 @@ class CacheUpdater( incrementalCacheArtifacts: Map, moduleNames: Map, rebuiltFileFragments: KotlinSourceFileMap - ): List { - stopwatch.startNext("Incremental cache - committing artifacts") - - val artifacts = mapLibrariesInOrder { libFile -> - val incrementalCacheArtifact = incrementalCacheArtifacts[libFile] ?: notFoundIcError("cache artifact", libFile) - val library = libraries[libFile] ?: notFoundIcError("library", libFile) + ): List = stopwatch.measure("Incremental cache - committing artifacts") { + incrementalCacheArtifacts.map { (libFile, incrementalCacheArtifact) -> incrementalCacheArtifact.buildModuleArtifactAndCommitCache( moduleName = moduleNames[libFile] ?: notFoundIcError("module name", libFile), - externalModuleName = library.jsOutputName, rebuiltFileFragments = rebuiltFileFragments[libFile] ?: emptyMap() ) } - - stopwatch.stop() - return artifacts } private fun compileDirtyFiles( compilerForIC: JsIrCompilerICInterface, loadedFragments: Map, dirtyFiles: Map> - ): MutableList JsIrProgramFragment>> { - stopwatch.startNext("Processing IR - lowering") - - val dirtyFilesForCompiling = mutableListOf() - val dirtyFilesForRestoring = mutableListOf>() - for ((libFile, libFragment) in loadedFragments) { - val dirtySrcFiles = dirtyFiles[libFile] ?: continue - for (irFile in libFragment.files) { - val srcFile = KotlinSourceFile(irFile) - if (srcFile in dirtySrcFiles) { - dirtyFilesForCompiling += irFile - dirtyFilesForRestoring += libFile to srcFile + ): MutableList JsIrProgramFragment>> = + stopwatch.measure("Processing IR - lowering") { + val dirtyFilesForCompiling = mutableListOf() + val dirtyFilesForRestoring = mutableListOf>() + for ((libFile, libFragment) in loadedFragments) { + val dirtySrcFiles = dirtyFiles[libFile] ?: continue + for (irFile in libFragment.files) { + val srcFile = KotlinSourceFile(irFile) + if (srcFile in dirtySrcFiles) { + dirtyFilesForCompiling += irFile + dirtyFilesForRestoring += libFile to srcFile + } } } - } - val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments) + val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments) - stopwatch.stop() - return dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile -> - Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i]) + dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile -> + Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i]) + } } - } private data class IrForDirtyFilesAndCompiler( val incrementalCacheArtifacts: Map, @@ -581,9 +562,6 @@ class CacheUpdater( ) private fun loadIrForDirtyFilesAndInitCompiler(): IrForDirtyFilesAndCompiler { - dirtyFileStats.clear() - - stopwatch.startNext("Incremental cache - initializing") val updater = CacheUpdaterInternal() stopwatch.startNext("Modified files - checking hashes and collecting") @@ -593,7 +571,7 @@ class CacheUpdater( val dirtyFileExports = updater.collectExportedSymbolsForDirtyFiles(modifiedFiles) stopwatch.startNext("Modified files - loading and linking IR") - val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, mainLibrary, dependencyGraph, irFactory()) + val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, updater.libraryDependencies, irFactory()) var loadedIr = jsIrLinkerLoader.loadIr(dirtyFileExports) var iterations = 0 @@ -630,8 +608,8 @@ class CacheUpdater( } stopwatch.startNext("Processing IR - initializing backend context") - val mainModule = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main lib loaded fragment", mainLibraryFile) - val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModule, compilerConfiguration) + val mainModuleFragment = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main module fragment", mainLibraryFile) + val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModuleFragment, compilerConfiguration) // Load declarations referenced during `context` initialization loadedIr.linker.loadUnboundSymbols(true) @@ -639,7 +617,7 @@ class CacheUpdater( val dirtyFiles = dirtyFileExports.entries.associateTo(HashMap(dirtyFileExports.size)) { it.key to HashSet(it.value.keys) } stopwatch.startNext("Processing IR - updating intrinsics and builtins dependencies") - updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModule, loadedIr.loadedFragments, dirtyFiles) + updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModuleFragment, loadedIr.loadedFragments, dirtyFiles) stopwatch.startNext("Incremental cache - building artifacts") val incrementalCachesArtifacts = updater.buildCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments) @@ -666,18 +644,19 @@ class CacheUpdater( private fun generateIrFragments( generators: MutableList JsIrProgramFragment>> - ): KotlinSourceFileMap { - stopwatch.startNext("Processing IR - generating program fragments") + ): KotlinSourceFileMap = stopwatch.measure("Processing IR - generating program fragments") { val rebuiltFragments = KotlinSourceFileMutableMap() while (generators.isNotEmpty()) { val (libFile, srcFile, fragmentGenerator) = generators.removeFirst() rebuiltFragments[libFile, srcFile] = fragmentGenerator() } - stopwatch.stop() - return rebuiltFragments + rebuiltFragments } fun actualizeCaches(): List { + stopwatch.clear() + dirtyFileStats.clear() + val (incrementalCachesArtifacts, moduleNames, generators) = loadIrAndMakeIrFragmentGenerators() val rebuiltFragments = generateIrFragments(generators) @@ -704,7 +683,7 @@ fun rebuildCacheForDirtyFiles( val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLibraryHeader(library).sourceFiles val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata }) - val jsIrLoader = JsIrLinkerLoader(configuration, library, dependencyGraph, irFactory) + val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, irFactory) val (jsIrLinker, irModules) = jsIrLoader.loadIr(KotlinSourceFileMap(modifiedFiles), true) val currentIrModule = irModules[libFile] ?: notFoundIcError("loaded fragment", libFile) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt index b05cc381fc5..494b3a83ceb 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt @@ -69,6 +69,12 @@ internal class StopwatchIC { val laps: List> get() = lapsImpl + fun clear() { + lapStart = 0 + lapDescription = null + lapsImpl.clear() + } + fun startNext(description: String) { val now = System.nanoTime() stop(now) @@ -82,4 +88,11 @@ internal class StopwatchIC { } lapDescription = null } + + inline fun measure(description: String, f: () -> T): T { + startNext(description) + val result = f() + stop() + return result + } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index b8cbc5b3b42..8967be19877 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer +import org.jetbrains.kotlin.ir.backend.js.jsOutputName import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.impl.javaFile @@ -85,7 +86,7 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile -> commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap()) } - return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts) + return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts, library.jsOutputName) } data class ModifiedFiles( diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt index 36ba4bf3e20..c9de562ec8a 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt @@ -54,11 +54,11 @@ internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val internal class IncrementalCacheArtifact( private val artifactsDir: File, private val forceRebuildJs: Boolean, - private val srcCacheActions: List + private val srcCacheActions: List, + private val externalModuleName: String? ) { fun buildModuleArtifactAndCommitCache( moduleName: String, - externalModuleName: String?, rebuiltFileFragments: Map, ): ModuleArtifact { val serializer = JsIrAstSerializer() diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt index e4015ce2b7a..202f33e6433 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt @@ -39,7 +39,6 @@ internal fun JsIrLinker.loadUnboundSymbols(checkNoUnbound: Boolean) { internal class JsIrLinkerLoader( private val compilerConfiguration: CompilerConfiguration, - private val library: KotlinLibrary, private val dependencyGraph: Map>, private val irFactory: IrFactory ) { @@ -87,11 +86,12 @@ internal class JsIrLinkerLoader( val loadedModules = loadModules() val jsIrLinker = createLinker(loadedModules) + val mainLibrary = dependencyGraph.keys.lastOrNull() ?: notFoundIcError("main library") val irModules = loadedModules.entries.associate { (descriptor, module) -> val libraryFile = KotlinLibraryFile(module) val modifiedStrategy = when { loadAllIr -> DeserializationStrategy.ALL - module == library -> DeserializationStrategy.ALL + module == mainLibrary -> DeserializationStrategy.ALL else -> DeserializationStrategy.EXPLICITLY_EXPORTED } val modified = modifiedFiles[libraryFile] ?: emptyMap() diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index df2f58deeb5..9b98544afc3 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -44,7 +44,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { companion object { private val TEST_DATA_DIR_PATH = System.getProperty("kotlin.js.test.root.out.dir") ?: error("'kotlin.js.test.root.out.dir' is not set") private const val BOX_FUNCTION_NAME = "box" - private const val STDLIB_ALIAS = "stdlib" private const val STDLIB_MODULE_NAME = "kotlin-kotlin-stdlib-js-ir" private val STDLIB_KLIB = File(System.getProperty("kotlin.js.stdlib.klib.path") ?: error("Please set stdlib path")).canonicalPath @@ -240,7 +239,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { ) val icCaches = cacheUpdater.actualizeCaches() - verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileStats(), testInfo) + verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileLastStats(), testInfo) val mainModuleName = icCaches.last().moduleExternalName val jsExecutableProducer = JsExecutableProducer( diff --git a/js/js.translator/testData/incremental/invalidation/removeModule/project.info b/js/js.translator/testData/incremental/invalidation/removeModule/project.info index ec60a5cb1eb..f1968258cbd 100644 --- a/js/js.translator/testData/incremental/invalidation/removeModule/project.info +++ b/js/js.translator/testData/incremental/invalidation/removeModule/project.info @@ -5,4 +5,4 @@ STEP 0: dirty js: lib1, lib2, main STEP 1: libs: lib1, lib2, main - dirty js: lib2 + dirty js: lib1, lib2