diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index ad3a44c1764..cf149dcf5cc 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -659,12 +659,12 @@ class K2JsIrCompiler : CLICompiler() { val artifacts = cacheUpdater.actualizeCaches() messageCollector.report(INFO, "IC rebuilt overall time: ${System.currentTimeMillis() - start}ms") - for ((event, duration) in cacheUpdater.getStopwatchLaps()) { + for ((event, duration) in cacheUpdater.getStopwatchLastLaps()) { messageCollector.report(INFO, " $event: ${(duration / 1e6).toInt()}ms") } var libIndex = 0 - for ((libFile, srcFiles) in cacheUpdater.getDirtyFileStats()) { + for ((libFile, srcFiles) in cacheUpdater.getDirtyFileLastStats()) { val (msg, showFiles) = when { srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) } -> "fully rebuilt due to clean build" to false srcFiles.values.all { it.contains(DirtyFileState.MODIFIED_CONFIG) } -> "fully rebuilt due to config modification" to false diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index a9dd86ce071..cd7a2343261 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -15,9 +15,9 @@ import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.konan.properties.propertyList import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder import org.jetbrains.kotlin.name.FqName import java.io.File -import java.security.MessageDigest import java.util.EnumSet fun interface JsIrCompilerICInterface { @@ -48,31 +48,24 @@ enum class DirtyFileState(val str: String) { class CacheUpdater( mainModule: String, - allModules: Collection, + private val allModules: Collection, private val icCacheRootDir: String, private val compilerConfiguration: CompilerConfiguration, private val irFactory: () -> IrFactory, private val mainArguments: List?, private val compilerInterfaceFactory: JsIrCompilerICInterfaceFactory ) { - private val stopwatch = StopwatchIC().apply { startNext("Loading klibs") } - - private val libraries = loadLibraries(allModules) - private val dependencyGraph = buildDependenciesGraph(libraries) - private val configHash = compilerConfiguration.configHashForIC() - - private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) - private val mainLibrary = libraries[mainLibraryFile] ?: notFoundIcError("main library", mainLibraryFile) + private val stopwatch = StopwatchIC() private val dirtyFileStats = KotlinSourceFileMutableMap>() - init { - stopwatch.stop() - } + private val configHash = compilerConfiguration.configHashForIC() - fun getDirtyFileStats(): KotlinSourceFileMap> = dirtyFileStats + private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) - fun getStopwatchLaps() = stopwatch.laps + fun getDirtyFileLastStats(): KotlinSourceFileMap> = dirtyFileStats + + fun getStopwatchLastLaps() = stopwatch.laps private fun MutableMap>.addDirtFileStat(srcFile: KotlinSourceFile, state: DirtyFileState) { when (val stats = this[srcFile]) { @@ -81,44 +74,41 @@ class CacheUpdater( } } - private fun loadLibraries(allModules: Collection): Map { - val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList() - val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger) - return allResolvedDependencies.getFullList().associateBy { KotlinLibraryFile(it) } - } - - private fun buildDependenciesGraph(libraries: Map): Map> { - val nameToKotlinLibrary = libraries.values.associateBy { it.moduleName } - return libraries.values.associateWith { - it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName -> - nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName") - } - } - } - - private fun mapLibrariesInOrder(f: (KotlinLibraryFile) -> T): List { - val visited = HashSet(libraries.size) - val result = ArrayList(libraries.size) - fun visitLibrary(lib: KotlinLibrary) { - if (visited.add(lib)) { - dependencyGraph[lib]?.forEach(::visitLibrary) - result += f(KotlinLibraryFile(lib)) - } - } - - visitLibrary(mainLibrary) - return result - } - private inner class CacheUpdaterInternal { val signatureHashCalculator = IdSignatureHashCalculator() - private val incrementalCaches = run { - val md5 = MessageDigest.getInstance("MD5") - libraries.entries.associate { (libFile, lib) -> + // libraries in topological order: [stdlib, ..., main] + val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") { + val repositories = compilerConfiguration[JSConfigurationKeys.REPOSITORIES] ?: emptyList() + val allResolvedDependencies = jsResolveLibraries(allModules, repositories, compilerConfiguration.resolverLogger) + + val libraries = allResolvedDependencies.getFullList(TopologicalLibraryOrder).let { resolvedLibraries -> + val mainLibraryIndex = resolvedLibraries.indexOfLast { + KotlinLibraryFile(it) == mainLibraryFile + }.takeIf { it >= 0 } ?: notFoundIcError("main library", mainLibraryFile) + + when (mainLibraryIndex) { + resolvedLibraries.lastIndex -> resolvedLibraries + else -> resolvedLibraries.filterIndexedTo(ArrayList(resolvedLibraries.size)) { index, _ -> + index != mainLibraryIndex + }.apply { add(resolvedLibraries[mainLibraryIndex]) } + } + } + + val nameToKotlinLibrary = libraries.associateBy { it.moduleName } + + libraries.associateWith { + it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName -> + nameToKotlinLibrary[depName] ?: notFoundIcError("library $depName") + } + } + } + + private val incrementalCaches = stopwatch.measure("Incremental cache - initializing") { + libraryDependencies.keys.associate { lib -> + val libFile = KotlinLibraryFile(lib) val file = File(libFile.path) - val hash = md5.digest(file.absolutePath.encodeToByteArray()).joinToString("", transform = { "%02x".format(it) }) - libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.name}.$hash")) + libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.absolutePath.stringHashForIC()}.${file.name}")) } } @@ -511,7 +501,8 @@ class CacheUpdater( jsIrLinker: JsIrLinker, loadedFragments: Map ): Map { - return mapLibrariesInOrder { libFile -> + return libraryDependencies.keys.associate { library -> + val libFile = KotlinLibraryFile(library) val incrementalCache = getLibIncrementalCache(libFile) val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile) val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor) @@ -520,7 +511,7 @@ class CacheUpdater( KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() } libFile to incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping) - }.toMap() + } } } @@ -528,50 +519,40 @@ class CacheUpdater( incrementalCacheArtifacts: Map, moduleNames: Map, rebuiltFileFragments: KotlinSourceFileMap - ): List { - stopwatch.startNext("Incremental cache - committing artifacts") - - val artifacts = mapLibrariesInOrder { libFile -> - val incrementalCacheArtifact = incrementalCacheArtifacts[libFile] ?: notFoundIcError("cache artifact", libFile) - val library = libraries[libFile] ?: notFoundIcError("library", libFile) + ): List = stopwatch.measure("Incremental cache - committing artifacts") { + incrementalCacheArtifacts.map { (libFile, incrementalCacheArtifact) -> incrementalCacheArtifact.buildModuleArtifactAndCommitCache( moduleName = moduleNames[libFile] ?: notFoundIcError("module name", libFile), - externalModuleName = library.jsOutputName, rebuiltFileFragments = rebuiltFileFragments[libFile] ?: emptyMap() ) } - - stopwatch.stop() - return artifacts } private fun compileDirtyFiles( compilerForIC: JsIrCompilerICInterface, loadedFragments: Map, dirtyFiles: Map> - ): MutableList JsIrProgramFragment>> { - stopwatch.startNext("Processing IR - lowering") - - val dirtyFilesForCompiling = mutableListOf() - val dirtyFilesForRestoring = mutableListOf>() - for ((libFile, libFragment) in loadedFragments) { - val dirtySrcFiles = dirtyFiles[libFile] ?: continue - for (irFile in libFragment.files) { - val srcFile = KotlinSourceFile(irFile) - if (srcFile in dirtySrcFiles) { - dirtyFilesForCompiling += irFile - dirtyFilesForRestoring += libFile to srcFile + ): MutableList JsIrProgramFragment>> = + stopwatch.measure("Processing IR - lowering") { + val dirtyFilesForCompiling = mutableListOf() + val dirtyFilesForRestoring = mutableListOf>() + for ((libFile, libFragment) in loadedFragments) { + val dirtySrcFiles = dirtyFiles[libFile] ?: continue + for (irFile in libFragment.files) { + val srcFile = KotlinSourceFile(irFile) + if (srcFile in dirtySrcFiles) { + dirtyFilesForCompiling += irFile + dirtyFilesForRestoring += libFile to srcFile + } } } - } - val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments) + val fragmentGenerators = compilerForIC.compile(loadedFragments.values, dirtyFilesForCompiling, mainArguments) - stopwatch.stop() - return dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile -> - Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i]) + dirtyFilesForRestoring.mapIndexedTo(ArrayList(dirtyFilesForRestoring.size)) { i, libFileAndSrcFile -> + Triple(libFileAndSrcFile.first, libFileAndSrcFile.second, fragmentGenerators[i]) + } } - } private data class IrForDirtyFilesAndCompiler( val incrementalCacheArtifacts: Map, @@ -581,9 +562,6 @@ class CacheUpdater( ) private fun loadIrForDirtyFilesAndInitCompiler(): IrForDirtyFilesAndCompiler { - dirtyFileStats.clear() - - stopwatch.startNext("Incremental cache - initializing") val updater = CacheUpdaterInternal() stopwatch.startNext("Modified files - checking hashes and collecting") @@ -593,7 +571,7 @@ class CacheUpdater( val dirtyFileExports = updater.collectExportedSymbolsForDirtyFiles(modifiedFiles) stopwatch.startNext("Modified files - loading and linking IR") - val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, mainLibrary, dependencyGraph, irFactory()) + val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, updater.libraryDependencies, irFactory()) var loadedIr = jsIrLinkerLoader.loadIr(dirtyFileExports) var iterations = 0 @@ -630,8 +608,8 @@ class CacheUpdater( } stopwatch.startNext("Processing IR - initializing backend context") - val mainModule = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main lib loaded fragment", mainLibraryFile) - val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModule, compilerConfiguration) + val mainModuleFragment = loadedIr.loadedFragments[mainLibraryFile] ?: notFoundIcError("main module fragment", mainLibraryFile) + val compilerForIC = compilerInterfaceFactory.createCompilerForIC(mainModuleFragment, compilerConfiguration) // Load declarations referenced during `context` initialization loadedIr.linker.loadUnboundSymbols(true) @@ -639,7 +617,7 @@ class CacheUpdater( val dirtyFiles = dirtyFileExports.entries.associateTo(HashMap(dirtyFileExports.size)) { it.key to HashSet(it.value.keys) } stopwatch.startNext("Processing IR - updating intrinsics and builtins dependencies") - updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModule, loadedIr.loadedFragments, dirtyFiles) + updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModuleFragment, loadedIr.loadedFragments, dirtyFiles) stopwatch.startNext("Incremental cache - building artifacts") val incrementalCachesArtifacts = updater.buildCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments) @@ -666,18 +644,19 @@ class CacheUpdater( private fun generateIrFragments( generators: MutableList JsIrProgramFragment>> - ): KotlinSourceFileMap { - stopwatch.startNext("Processing IR - generating program fragments") + ): KotlinSourceFileMap = stopwatch.measure("Processing IR - generating program fragments") { val rebuiltFragments = KotlinSourceFileMutableMap() while (generators.isNotEmpty()) { val (libFile, srcFile, fragmentGenerator) = generators.removeFirst() rebuiltFragments[libFile, srcFile] = fragmentGenerator() } - stopwatch.stop() - return rebuiltFragments + rebuiltFragments } fun actualizeCaches(): List { + stopwatch.clear() + dirtyFileStats.clear() + val (incrementalCachesArtifacts, moduleNames, generators) = loadIrAndMakeIrFragmentGenerators() val rebuiltFragments = generateIrFragments(generators) @@ -704,7 +683,7 @@ fun rebuildCacheForDirtyFiles( val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLibraryHeader(library).sourceFiles val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata }) - val jsIrLoader = JsIrLinkerLoader(configuration, library, dependencyGraph, irFactory) + val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, irFactory) val (jsIrLinker, irModules) = jsIrLoader.loadIr(KotlinSourceFileMap(modifiedFiles), true) val currentIrModule = irModules[libFile] ?: notFoundIcError("loaded fragment", libFile) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt index b05cc381fc5..494b3a83ceb 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/ICUtils.kt @@ -69,6 +69,12 @@ internal class StopwatchIC { val laps: List> get() = lapsImpl + fun clear() { + lapStart = 0 + lapDescription = null + lapsImpl.clear() + } + fun startNext(description: String) { val now = System.nanoTime() stop(now) @@ -82,4 +88,11 @@ internal class StopwatchIC { } lapDescription = null } + + inline fun measure(description: String, f: () -> T): T { + startNext(description) + val result = f() + stop() + return result + } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index b8cbc5b3b42..8967be19877 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer +import org.jetbrains.kotlin.ir.backend.js.jsOutputName import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.impl.javaFile @@ -85,7 +86,7 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile -> commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap()) } - return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts) + return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts, library.jsOutputName) } data class ModifiedFiles( diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt index 36ba4bf3e20..c9de562ec8a 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt @@ -54,11 +54,11 @@ internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val internal class IncrementalCacheArtifact( private val artifactsDir: File, private val forceRebuildJs: Boolean, - private val srcCacheActions: List + private val srcCacheActions: List, + private val externalModuleName: String? ) { fun buildModuleArtifactAndCommitCache( moduleName: String, - externalModuleName: String?, rebuiltFileFragments: Map, ): ModuleArtifact { val serializer = JsIrAstSerializer() diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt index e4015ce2b7a..202f33e6433 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/JsIrLinkerLoader.kt @@ -39,7 +39,6 @@ internal fun JsIrLinker.loadUnboundSymbols(checkNoUnbound: Boolean) { internal class JsIrLinkerLoader( private val compilerConfiguration: CompilerConfiguration, - private val library: KotlinLibrary, private val dependencyGraph: Map>, private val irFactory: IrFactory ) { @@ -87,11 +86,12 @@ internal class JsIrLinkerLoader( val loadedModules = loadModules() val jsIrLinker = createLinker(loadedModules) + val mainLibrary = dependencyGraph.keys.lastOrNull() ?: notFoundIcError("main library") val irModules = loadedModules.entries.associate { (descriptor, module) -> val libraryFile = KotlinLibraryFile(module) val modifiedStrategy = when { loadAllIr -> DeserializationStrategy.ALL - module == library -> DeserializationStrategy.ALL + module == mainLibrary -> DeserializationStrategy.ALL else -> DeserializationStrategy.EXPLICITLY_EXPORTED } val modified = modifiedFiles[libraryFile] ?: emptyMap() diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index df2f58deeb5..9b98544afc3 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -44,7 +44,6 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { companion object { private val TEST_DATA_DIR_PATH = System.getProperty("kotlin.js.test.root.out.dir") ?: error("'kotlin.js.test.root.out.dir' is not set") private const val BOX_FUNCTION_NAME = "box" - private const val STDLIB_ALIAS = "stdlib" private const val STDLIB_MODULE_NAME = "kotlin-kotlin-stdlib-js-ir" private val STDLIB_KLIB = File(System.getProperty("kotlin.js.stdlib.klib.path") ?: error("Please set stdlib path")).canonicalPath @@ -240,7 +239,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { ) val icCaches = cacheUpdater.actualizeCaches() - verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileStats(), testInfo) + verifyCacheUpdateStats(projStep.id, cacheUpdater.getDirtyFileLastStats(), testInfo) val mainModuleName = icCaches.last().moduleExternalName val jsExecutableProducer = JsExecutableProducer( diff --git a/js/js.translator/testData/incremental/invalidation/removeModule/project.info b/js/js.translator/testData/incremental/invalidation/removeModule/project.info index ec60a5cb1eb..f1968258cbd 100644 --- a/js/js.translator/testData/incremental/invalidation/removeModule/project.info +++ b/js/js.translator/testData/incremental/invalidation/removeModule/project.info @@ -5,4 +5,4 @@ STEP 0: dirty js: lib1, lib2, main STEP 1: libs: lib1, lib2, main - dirty js: lib2 + dirty js: lib1, lib2