From 43a0876c2619121f577ff13d0e6329f79254b941 Mon Sep 17 00:00:00 2001 From: Alexander Korepanov Date: Thu, 10 Feb 2022 12:23:24 +0300 Subject: [PATCH] [JS IR] IC invalidation performance improvements - Cache signature readers and deserializers - Cache inline function hashes - IC refactoring - [gradle] Use hash of module path for cache dir; Fix KT-51238 ^KT-51238 Fixed --- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 19 +- .../kotlin/ir/backend/js/ic/CacheInfo.kt | 36 +++ .../ir/backend/js/ic/CacheLazyLoader.kt | 86 ++++++ .../kotlin/ir/backend/js/ic/fileUtil.kt | 45 --- .../kotlin/ir/backend/js/ic/invalidation.kt | 275 +++++++----------- .../kotlin/ir/backend/js/ic/CacheAccessors.kt | 8 + .../incremental/AbstractInvalidationTest.kt | 36 +-- .../test/utils/JsIrIncrementalDataProvider.kt | 3 + .../InvalidationTestGenerated.java | 5 + .../mainModuleInvalidation/lib1/l1.kt | 1 + .../mainModuleInvalidation/lib1/module.info | 6 + .../mainModuleInvalidation/main/m.kt | 2 + .../mainModuleInvalidation/main/m.kt.1.txt | 2 + .../mainModuleInvalidation/main/m0.kt | 2 + .../mainModuleInvalidation/main/m1.kt | 2 + .../mainModuleInvalidation/main/module.info | 9 + .../mainModuleInvalidation/project.info | 7 + .../removeUnusedFile/lib1/module.info | 3 + .../removeUnusedFile/main/module.info | 5 +- .../removeUnusedFile/project.info | 5 +- .../gradle/targets/js/ir/KotlinJsIrLink.kt | 11 +- .../kotlin/gradle/utils/cacheKlibUtils.kt | 4 +- 22 files changed, 315 insertions(+), 257 deletions(-) create mode 100644 compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheInfo.kt create mode 100644 compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheLazyLoader.kt delete mode 100644 compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/l1.kt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/module.info create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt.1.txt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m0.kt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m1.kt create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/module.info create mode 100644 js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/project.info diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index 00431d2b3ba..831ad4db9ca 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -43,9 +43,7 @@ import org.jetbrains.kotlin.incremental.js.IncrementalResultsConsumer import org.jetbrains.kotlin.backend.wasm.dce.eliminateDeadDeclarations import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity -import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCaches -import org.jetbrains.kotlin.ir.backend.js.ic.buildCacheForModuleFiles -import org.jetbrains.kotlin.ir.backend.js.ic.loadModuleCaches +import org.jetbrains.kotlin.ir.backend.js.ic.* import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformer import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.IrModuleToJsTransformerTmp import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.SourceMapsInfo @@ -210,7 +208,7 @@ class K2JsIrCompiler : CLICompiler() { val includes = arguments.includes!! - val start = System.currentTimeMillis() + var start = System.currentTimeMillis() actualizeCaches( includes, @@ -220,12 +218,15 @@ class K2JsIrCompiler : CLICompiler() { { IrFactoryImplForJsIC(WholeWorldStageController()) }, mainCallArguments, ::buildCacheForModuleFiles, - ) { updateStatus -> - if (updateStatus.upToDate) { - messageCollector.report(INFO, "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms") - } else { - messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms") + ) { updateStatus, updatedModule -> + val now = System.currentTimeMillis() + val strStatus = when (updateStatus) { + CacheUpdateStatus.FAST_PATH -> "up-to-date; fast check" + CacheUpdateStatus.NO_DIRTY_FILES -> "up-to-date; full check" + CacheUpdateStatus.DIRTY -> "dirty; cache building" } + messageCollector.report(INFO, "IC per-file is $strStatus duration ${now - start}ms; module [${File(updatedModule).name}]") + start = now } } else emptyList() diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheInfo.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheInfo.kt new file mode 100644 index 00000000000..5b7ec175ee4 --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheInfo.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import java.io.File + +// TODO md5 hash +data class CacheInfo( + val path: String, + val libPath: String, + val moduleName: String?, + var flatHash: ULong, + var transHash: ULong, + var configHash: ULong +) { + companion object { + fun load(path: String): CacheInfo? { + val info = File(File(path), "info") + + if (!info.exists()) return null + + val (libPath, moduleName, flatHash, transHash, configHash) = info.readLines() + + // safe cast for the backward compatibility with the cache from the previous compiler versions + val configHashULong = configHash.toULongOrNull(16) ?: 0UL + return CacheInfo(path, libPath, moduleName, flatHash.toULong(16), transHash.toULong(16), configHashULong) + } + + fun loadOrCreate(path: String, libPath: String): CacheInfo { + return load(path) ?: CacheInfo(path, libPath, null, 0UL, 0UL, 0UL) + } + } +} diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheLazyLoader.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheLazyLoader.kt new file mode 100644 index 00000000000..e199001bb5e --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheLazyLoader.kt @@ -0,0 +1,86 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer +import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource +import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes +import org.jetbrains.kotlin.backend.common.serialization.codedInputStream +import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite + + +class CacheLazyLoader(private val cacheProvider: PersistentCacheProvider, private val library: KotlinLibrary) { + private val graphInlineCache = mutableMapOf>>() + private val fingerPrintCache = mutableMapOf() + + val signatureReadersList: List> by lazy { + library.filesAndSigReaders() + } + + private val signatureReadersMap: Map by lazy { + signatureReadersList.toMap() + } + + val allInlineFunctionHashes: Map by lazy { + cacheProvider.allInlineHashes { librarySrc, index -> + val libReader = signatureReadersMap[librarySrc] ?: error("No module reader for lib $librarySrc") + libReader.deserializeIdSignature(index) + } + } + + fun getInlineHashesByFile() = signatureReadersList.associateTo(mutableMapOf()) { + it.first to cacheProvider.inlineHashes(it.first) { index -> it.second.deserializeIdSignature(index) } + } + + fun getInlineGraphForFile(srcFile: String) = graphInlineCache.getOrPut(srcFile) { + val fileReader = signatureReadersMap[srcFile] ?: error("Cannot find signature reader for $srcFile") + cacheProvider.inlineGraphForFile(srcFile) { index -> fileReader.deserializeIdSignature(index) } + } + + fun getFileFingerPrint(srcFile: String) = fingerPrintCache.getOrPut(srcFile) { + cacheProvider.fileFingerPrint(srcFile) + } + + fun getFilePaths() = cacheProvider.filePaths() + + fun clearCaches() { + graphInlineCache.clear() + fingerPrintCache.clear() + } + + private fun KotlinLibrary.filesAndSigReaders(): List> { + val fileSize = fileCount() + val result = ArrayList>(fileSize) + val extReg = ExtensionRegistryLite.newInstance() + + for (i in 0 until fileSize) { + val fileStream = file(i).codedInputStream + val fileProto = IrFile.parseFrom(fileStream, extReg) + val sigReader = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() { + private fun err(): Nothing = error("Not supported") + override fun irDeclaration(index: Int): ByteArray = err() + + override fun type(index: Int): ByteArray = err() + + override fun signature(index: Int): ByteArray = signature(index, i) + + override fun string(index: Int): ByteArray = string(index, i) + + override fun body(index: Int): ByteArray = err() + + override fun debugInfo(index: Int): ByteArray? = null + }), null) + + result.add(fileProto.fileEntry.name to sigReader) + } + + return result + } +} + diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt deleted file mode 100644 index c2773eaa7d6..00000000000 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. - * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. - */ - -package org.jetbrains.kotlin.ir.backend.js.ic - -import java.io.File -import java.io.PrintWriter - -// TODO md5 hash -data class CacheInfo(val path: String, val libPath: String, var flatHash: ULong, var transHash: ULong, var configHash: ULong) { - fun save() { - PrintWriter(File(File(path), "info")).use { - it.println(libPath) - it.println(flatHash.toString(16)) - it.println(transHash.toString(16)) - it.println(configHash.toString(16)) - } - } - - companion object { - fun load(path: String): CacheInfo? { - val info = File(File(path), "info") - - if (!info.exists()) return null - - val (libPath, _, flatHash, transHash, configHash) = info.readLines() - - // safe cast for the backward compatibility with the cache from the previous compiler versions - val configHashULong = configHash.toULongOrNull(16) ?: 0UL - return CacheInfo(path, libPath, flatHash.toULong(16), transHash.toULong(16), configHashULong) - } - - fun loadOrCreate( - path: String, - moduleName: String, - flatHash: ULong = 0UL, - transHash: ULong = 0UL, - configHash: ULong = 0UL - ): CacheInfo { - return load(path) ?: CacheInfo(path, moduleName, flatHash, transHash, configHash) - } - } -} diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt index f4e98c1561a..9065e3c1f78 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt @@ -5,10 +5,6 @@ package org.jetbrains.kotlin.ir.backend.js.ic -import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer -import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource -import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes -import org.jetbrains.kotlin.backend.common.serialization.codedInputStream import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureDescriptor import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.LanguageVersionSettings @@ -32,12 +28,10 @@ import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.unresolvedDependencies import org.jetbrains.kotlin.name.FqName -import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl import org.jetbrains.kotlin.storage.LockBasedStorageManager import java.io.File import java.security.MessageDigest -import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile private fun KotlinLibrary.fingerprint(fileIndex: Int): Hash { return ((((types(fileIndex).md5() * 31) + signatures(fileIndex).md5()) * 31 + strings(fileIndex).md5()) * 31 + declarations(fileIndex).md5()) * 31 + bodies( @@ -50,11 +44,9 @@ private fun invalidateCacheForModule( libraryFiles: List, externalHashes: Map, cachedInlineHashesForFile: MutableMap>, - cacheProvider: PersistentCacheProvider, + cacheProvider: CacheLazyLoader, cacheConsumer: PersistentCacheConsumer, - signatureResolver: (String, Int) -> IdSignature, - fileFingerPrints: MutableMap, - configUpdated: Boolean + fileFingerPrints: MutableMap ): Pair, Collection> { val dirtyFiles = mutableSetOf() @@ -62,12 +54,12 @@ private fun invalidateCacheForModule( for ((index, file) in libraryFiles.withIndex()) { // 1. get cached fingerprints - val fileOldFingerprint = cacheProvider.fileFingerPrint(file) + val fileOldFingerprint = cacheProvider.getFileFingerPrint(file) // 2. calculate new fingerprints val fileNewFingerprint = library.fingerprint(index) - if (fileOldFingerprint != fileNewFingerprint || configUpdated) { + if (fileOldFingerprint != fileNewFingerprint) { fileFingerPrints[file] = fileNewFingerprint cachedInlineHashesForFile.remove(file) @@ -77,33 +69,26 @@ private fun invalidateCacheForModule( } // 4. extend dirty set with inline functions - - val graphCache = mutableMapOf>>() - - var oldSize: Int do { - oldSize = dirtyFiles.size - if (dirtyFiles.size == libraryFiles.size) break + val oldSize = dirtyFiles.size for (file in libraryFiles) { if (file in dirtyFiles) continue // check for clean file - val inlineGraph = graphCache.getOrPut(file) { cacheProvider.inlineGraphForFile(file) { signatureResolver(file, it) } } + val inlineGraph = cacheProvider.getInlineGraphForFile(file) for ((sig, oldHash) in inlineGraph) { val actualHash = externalHashes[sig] ?: cachedInlineHashesForFile.values.firstNotNullOfOrNull { it[sig] } - // null means inline function is from dirty file, could be a bit more optimal - if (actualHash != null) { - if (oldHash == actualHash) continue + if (actualHash == null || oldHash != actualHash) { + cachedInlineHashesForFile.remove(file) + dirtyFiles.add(file) + fileFingerPrints[file] = cacheProvider.getFileFingerPrint(file) + break } - - cachedInlineHashesForFile.remove(file) - dirtyFiles.add(file) - fileFingerPrints[file] = cacheProvider.fileFingerPrint(file) } } } while (oldSize != dirtyFiles.size) @@ -113,47 +98,28 @@ private fun invalidateCacheForModule( cacheConsumer.invalidateForFile(dirty) } - val cachedFiles = cacheProvider.filePaths() + val cachedFiles = cacheProvider.getFilePaths() val deletedFiles = cachedFiles - libraryFiles.toSet() for (deleted in deletedFiles) { cacheConsumer.invalidateForFile(deleted) } + cacheProvider.clearCaches() return dirtyFiles to deletedFiles } -private fun KotlinLibrary.filesAndSigReaders(): List> { - val fileSize = fileCount() - val result = ArrayList>(fileSize) - val extReg = ExtensionRegistryLite.newInstance() - - for (i in 0 until fileSize) { - val fileStream = file(i).codedInputStream - val fileProto = ProtoFile.parseFrom(fileStream, extReg) - val sigReader = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() { - private fun err(): Nothing = error("Not supported") - override fun irDeclaration(index: Int): ByteArray = err() - - override fun type(index: Int): ByteArray = err() - - override fun signature(index: Int): ByteArray = signature(index, i) - - override fun string(index: Int): ByteArray = string(index, i) - - override fun body(index: Int): ByteArray = err() - - override fun debugInfo(index: Int): ByteArray? = null - }), null) - - result.add(fileProto.fileEntry.name to sigReader) - } - - return result +private fun CacheInfo.commitLibraryInfo(cacheConsumer: PersistentCacheConsumer, newModuleName: String? = null) { + cacheConsumer.commitLibraryInfo( + libPath.toCanonicalPath(), + newModuleName ?: moduleName ?: error("Cannot find module name for $libPath module"), + flatHash, + transHash, + configHash + ) } private fun buildCacheForModule( - libraryInfo: CacheInfo, configuration: CompilerConfiguration, irModule: IrModuleFragment, deserializer: JsIrLinker, @@ -217,14 +183,6 @@ private fun buildCacheForModule( emptySet(), mainArguments ) - - cacheConsumer.commitLibraryInfo( - libraryInfo.libPath.toCanonicalPath(), - irModule.name.asString(), - libraryInfo.flatHash, - libraryInfo.transHash, - libraryInfo.configHash - ) } private fun loadModules( @@ -303,6 +261,10 @@ fun loadCacheInfo(cachePaths: Collection): MutableMap): Map { val allResolvedDependencies = jsResolveLibraries( dependencies, @@ -310,11 +272,9 @@ private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Co configuration[IrMessageLogger.IR_MESSAGE_LOGGER].toResolverLogger() ) - return allResolvedDependencies.getFullList().associateBy { it.libraryFile.path.toCanonicalPath() } + return allResolvedDependencies.getFullList().associateBy { it.moduleCanonicalName() } } -fun String.toCanonicalPath(): String = File(this).canonicalPath - typealias ModuleName = String typealias ModulePath = String typealias FilePath = String @@ -375,22 +335,20 @@ private fun CompilerConfiguration.calcMD5(): ULong { } private fun checkLibrariesHash( - libraries: Map, + currentLib: KotlinLibrary, dependencyGraph: Map>, + currentCache: CacheInfo, icCacheMap: Map, modulePath: ModulePath ): Boolean { - val currentLib = libraries[modulePath] ?: error("1") - val currentCache = icCacheMap[modulePath] ?: error("2") - val flatHash = File(modulePath).md5() - val dependencies = dependencyGraph[currentLib] ?: error("3") + val dependencies = dependencyGraph[currentLib] ?: error("Cannot find dependencies for ${currentLib.libraryName}") var transHash = flatHash for (dep in dependencies) { - val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("4") + val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("Cannot cache info for ${dep.libraryName}") transHash += depCache.transHash } @@ -403,11 +361,19 @@ private fun checkLibrariesHash( return true } +private fun CacheInfo.invalidateCacheForNewConfig(configMD5: ULong, cacheConsumer: PersistentCacheConsumer) { + if (configHash != configMD5) { + cacheConsumer.invalidate() + flatHash = 0UL + transHash = 0UL + configHash = configMD5 + } +} + enum class CacheUpdateStatus(val upToDate: Boolean) { DIRTY(upToDate = false), NO_DIRTY_FILES(upToDate = true), FAST_PATH(upToDate = true) - } fun actualizeCaches( @@ -418,13 +384,14 @@ fun actualizeCaches( irFactory: () -> IrFactory, mainArguments: List?, executor: CacheExecutor, - callback: (CacheUpdateStatus) -> Unit + callback: (CacheUpdateStatus, String) -> Unit ): List { val (libraries, dependencyGraph, configMD5) = CacheConfiguration(dependencies, compilerConfiguration) val cacheMap = libraries.values.zip(icCachePaths).toMap() - val icCacheMap: MutableMap = mutableMapOf() + val icCacheMap = mutableMapOf() val resultCaches = mutableListOf() + val persistentCacheProviders = mutableMapOf() val visitedLibraries = mutableSetOf() fun visitDependency(library: KotlinLibrary) { @@ -437,23 +404,30 @@ fun actualizeCaches( val cachePath = cacheMap[library] ?: error("Unknown cache for library ${library.libraryName}") resultCaches.add(cachePath) - val moduleName = library.libraryFile.path.toCanonicalPath() - val cacheInfo = CacheInfo.loadOrCreate(cachePath, moduleName, configHash = configMD5) - icCacheMap[moduleName] = cacheInfo + val modulePath = library.moduleCanonicalName() + val cacheInfo = CacheInfo.loadOrCreate(cachePath, modulePath) + icCacheMap[modulePath] = cacheInfo - val updateStatus = actualizeCacheForModule( - moduleName = moduleName, - cachePath = cachePath, - compilerConfiguration = compilerConfiguration, - configMD5 = configMD5, - libraries = libraries, - dependencyGraph = dependencyGraph, - icCacheMap = icCacheMap, - irFactory = irFactory(), - mainArguments = mainArguments, - executor = executor - ) - callback(updateStatus) + val cacheConsumer = createCacheConsumer(cachePath) + persistentCacheProviders[library] = CacheLazyLoader(createCacheProvider(cachePath), library) + + cacheInfo.invalidateCacheForNewConfig(configMD5, cacheConsumer) + val updateStatus = when { + checkLibrariesHash(library, dependencyGraph, cacheInfo, icCacheMap, modulePath) -> CacheUpdateStatus.FAST_PATH + else -> actualizeCacheForModule( + library = library, + libraryInfo = cacheInfo, + configuration = compilerConfiguration, + dependencyGraph = getDependencySubGraphFor(library, dependencyGraph), + persistentCacheProviders = persistentCacheProviders, + persistentCacheConsumer = cacheConsumer, + irFactory = irFactory(), + mainArguments = mainArguments, + cacheExecutor = executor, + ) + } + + callback(updateStatus, modulePath) } val canonicalIncludes = includes.toCanonicalPath() @@ -462,6 +436,26 @@ fun actualizeCaches( return resultCaches } +private fun getDependencySubGraphFor( + targetLib: KotlinLibrary, + dependencyGraph: Map> +): Map> { + val subGraph = mutableMapOf>() + + fun addDependsFor(library: KotlinLibrary) { + if (library in subGraph) { + return + } + val dependencies = dependencyGraph[library] ?: error("Cannot find dependencies for ${library.libraryName}") + subGraph[library] = dependencies + for (dependency in dependencies) { + addDependsFor(dependency) + } + } + addDependsFor(targetLib) + return subGraph +} + class CacheConfiguration( private val dependencies: Collection, val compilerConfiguration: CompilerConfiguration @@ -487,97 +481,32 @@ class CacheConfiguration( operator fun component3() = configMD5 } -// Returns true if caches up-to-date -fun actualizeCacheForModule( - moduleName: String, - cachePath: String, - compilerConfiguration: CompilerConfiguration, - configMD5: ULong, - libraries: Map, - dependencyGraph: Map>, - icCacheMap: Map, - irFactory: IrFactory, - mainArguments: List?, - executor: CacheExecutor -): CacheUpdateStatus { - val cacheInfo = icCacheMap[moduleName] ?: error("Cache for $moduleName not found") - - val configUpdated = configMD5 != cacheInfo.configHash - cacheInfo.configHash = configMD5 - if (checkLibrariesHash(libraries, dependencyGraph, icCacheMap, moduleName) && !configUpdated) { - return CacheUpdateStatus.FAST_PATH // up-to-date - } - - val persistentCacheProviders = icCacheMap.map { (lib, cache) -> - libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache.path) - }.toMap() - - val currentModule = libraries[moduleName] ?: error("No loaded library found for path $moduleName") - val persistentCacheConsumer = createCacheConsumer(cachePath) - - return actualizeCacheForModule( - currentModule, - cacheInfo, - compilerConfiguration, - dependencyGraph, - persistentCacheProviders, - persistentCacheConsumer, - irFactory, - mainArguments, - executor, - configUpdated - ) -} - - private fun actualizeCacheForModule( library: KotlinLibrary, libraryInfo: CacheInfo, configuration: CompilerConfiguration, dependencyGraph: Map>, - persistentCacheProviders: Map, + persistentCacheProviders: Map, persistentCacheConsumer: PersistentCacheConsumer, irFactory: IrFactory, mainArguments: List?, cacheExecutor: CacheExecutor, - configUpdated: Boolean ): CacheUpdateStatus { // 1. Invalidate val dependencies = dependencyGraph[library]!! - val filesAndSigReaders = library.filesAndSigReaders() - val signatureDeserializers = filesAndSigReaders.toMap() - val libraryFiles = filesAndSigReaders.map { it.first } - - val depReaders = dependencies.associateWith { it.filesAndSigReaders().toMap() } - - val signatureResolver: (String, Int) -> IdSignature = { f, s -> - signatureDeserializers[f]?.deserializeIdSignature(s) ?: error("Cannot deserialize sig $s from $f") - } + val currentLibraryCacheProvider = persistentCacheProviders[library] ?: error("No cache provider for $library") + val libraryFiles = currentLibraryCacheProvider.signatureReadersList.map { it.first } val sigHashes = mutableMapOf() dependencies.forEach { lib -> persistentCacheProviders[lib]?.let { provider -> - val moduleReaders = depReaders[lib]!! - val inlineHashes = provider.allInlineHashes { f, i -> - val moduleReader = moduleReaders[f] - ?: error("No module reader for file $f") - moduleReader.deserializeIdSignature(i) - } - sigHashes.putAll(inlineHashes) + sigHashes.putAll(provider.allInlineFunctionHashes) } } val fileFingerPrints = mutableMapOf() - - val currentLibraryCacheProvider = persistentCacheProviders[library] ?: error("No cache provider for $library") - - val fileCachedInlineHashes = mutableMapOf>() - - filesAndSigReaders.forEach { (filePath, sigReader) -> - fileCachedInlineHashes[filePath] = - currentLibraryCacheProvider.inlineHashes(filePath) { s -> sigReader.deserializeIdSignature(s) } - } + val fileCachedInlineHashes = currentLibraryCacheProvider.getInlineHashesByFile() val (dirtySet, deletedFiles) = invalidateCacheForModule( library, @@ -586,12 +515,14 @@ private fun actualizeCacheForModule( fileCachedInlineHashes, currentLibraryCacheProvider, persistentCacheConsumer, - signatureResolver, - fileFingerPrints, - configUpdated + fileFingerPrints ) - if (dirtySet.isEmpty()) return CacheUpdateStatus.NO_DIRTY_FILES // up-to-date + if (dirtySet.isEmpty()) { + // up-to-date + libraryInfo.commitLibraryInfo(persistentCacheConsumer) + return CacheUpdateStatus.NO_DIRTY_FILES + } // 2. Build @@ -619,17 +550,13 @@ private fun actualizeCacheForModule( val currentIrModule = irModules.find { it.second == library }?.first!! val currentModuleDeserializer = jsIrLinker.moduleDeserializer(currentIrModule.descriptor) - for (file in libraryFiles) { - if (file !in dirtySet) { - val sigDeserializer = signatureDeserializers[file]!! - sigHashes.putAll(currentLibraryCacheProvider.inlineHashes(file) { sigDeserializer.deserializeIdSignature(it) }) - } + for (hashes in fileCachedInlineHashes.values) { + sigHashes.putAll(hashes) } val deserializers = dirtySet.associateWith { currentModuleDeserializer.signatureDeserializerForFile(it).signatureToIndexMapping() } buildCacheForModule( - libraryInfo, configuration, currentIrModule, jsIrLinker, @@ -643,6 +570,8 @@ private fun actualizeCacheForModule( mainArguments, cacheExecutor ) + + libraryInfo.commitLibraryInfo(persistentCacheConsumer, currentIrModule.name.asString()) return CacheUpdateStatus.DIRTY // invalidated and re-built } @@ -684,7 +613,7 @@ fun rebuildCacheForDirtyFiles( val currentIrModule = irModules.find { it.second == library }?.first!! - cacheConsumer.commitLibraryInfo(library.libraryFile.path.toCanonicalPath(), currentIrModule.name.asString(), 0UL, 0UL, 0UL) + cacheConsumer.commitLibraryInfo(library.moduleCanonicalName(), currentIrModule.name.asString(), 0UL, 0UL, 0UL) buildCacheForModuleFiles( currentIrModule, @@ -739,4 +668,4 @@ fun loadModuleCaches(icCachePaths: Collection): Map f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f)) }) } -} \ No newline at end of file +} diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt index e0bdc4af6c0..aa727503476 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt @@ -202,6 +202,7 @@ interface PersistentCacheConsumer { fun commitBinaryDts(path: String, dstData: ByteArray) fun commitSourceMap(path: String, mapData: ByteArray) fun invalidateForFile(path: String) + fun invalidate() fun commitLibraryInfo(libraryPath: String, moduleName: String, flatHash: ULong, transHash: ULong, configHash: ULong) @@ -227,6 +228,9 @@ interface PersistentCacheConsumer { override fun invalidateForFile(path: String) { } + override fun invalidate() { + } + override fun commitBinaryAst(path: String, astData: ByteArray) { } @@ -303,6 +307,10 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac //fileDir.deleteRecursively() } + override fun invalidate() { + File(cachePath).deleteRecursively() + } + private fun commitByteArrayToCacheFile(fileName: String, cacheName: String, data: ByteArray) { val fileId = createFileCacheId(fileName) val cacheDir = File(File(cachePath), fileId) diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index a4277919289..b2adce0d5d8 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -254,29 +254,25 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { mainArguments: List?, executor: CacheExecutor ): CacheUpdateStatus { - val (libraries, dependencyGraph, configMD5) = CacheConfiguration(dependencies, compilerConfiguration) - val modulePath = moduleName.toCanonicalPath() + val icCacheMap: Map = loadCacheInfo(icCachePaths).also { - it[modulePath] = CacheInfo.loadOrCreate( - cachePath, - moduleName, - configHash = configMD5 - ) + it[modulePath] = CacheInfo.loadOrCreate(cachePath, modulePath) } - return actualizeCacheForModule( - moduleName = modulePath, - cachePath = cachePath, - compilerConfiguration = compilerConfiguration, - configMD5 = configMD5, - libraries = libraries, - dependencyGraph = dependencyGraph, - icCacheMap = icCacheMap, - irFactory = irFactory, - mainArguments = mainArguments, - executor = executor - ) + val statuses = mutableMapOf() + actualizeCaches( + modulePath, + compilerConfiguration, + dependencies, + icCacheMap.map { it.value.path }, + { irFactory }, + mainArguments, + executor + ) { updateStatus, updatedModule -> + statuses[updatedModule] = updateStatus + } + return statuses[modulePath] ?: error("Status is missed for $modulePath") } private fun KotlinCoreEnvironment.createPsiFile(file: File): KtFile { @@ -347,4 +343,4 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { const val TEST_DATA_DIR_PATH = "js/js.translator/testData/" } -} \ No newline at end of file +} diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt index cfea5c6ca88..58dce8997b7 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt @@ -112,6 +112,9 @@ class TestModuleCache(val files: MutableMap) { files.remove(path) } + override fun invalidate() { + } + override fun commitLibraryInfo(libraryPath: String, moduleName: String, flatHash: ULong, transHash: ULong, configHash: ULong) { storedModuleName = moduleName } diff --git a/js/js.tests/tests-gen/org/jetbrains/kotlin/incremental/InvalidationTestGenerated.java b/js/js.tests/tests-gen/org/jetbrains/kotlin/incremental/InvalidationTestGenerated.java index 9cdc8f1218c..1ea3f071736 100644 --- a/js/js.tests/tests-gen/org/jetbrains/kotlin/incremental/InvalidationTestGenerated.java +++ b/js/js.tests/tests-gen/org/jetbrains/kotlin/incremental/InvalidationTestGenerated.java @@ -100,6 +100,11 @@ public class InvalidationTestGenerated extends AbstractInvalidationTest { runTest("js/js.translator/testData/incremental/invalidation/jsExport/"); } + @TestMetadata("mainModuleInvalidation") + public void testMainModuleInvalidation() throws Exception { + runTest("js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/"); + } + @TestMetadata("moveFilesBetweenModules") public void testMoveFilesBetweenModules() throws Exception { runTest("js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/"); diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/l1.kt b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/l1.kt new file mode 100644 index 00000000000..8e6e4306513 --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/l1.kt @@ -0,0 +1 @@ +fun qux() = 42 diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/module.info b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/module.info new file mode 100644 index 00000000000..f500435de26 --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/lib1/module.info @@ -0,0 +1,6 @@ +STEP 0: + dependencies: stdlib + dirty: l1.kt +STEP 1: + dependencies: stdlib + flags: FP diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt new file mode 100644 index 00000000000..3ed427f027a --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt @@ -0,0 +1,2 @@ + +fun box() = qux() \ No newline at end of file diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt.1.txt b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt.1.txt new file mode 100644 index 00000000000..9a4e0081659 --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m.kt.1.txt @@ -0,0 +1,2 @@ + +fun box() = qux() + 88 diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m0.kt b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m0.kt new file mode 100644 index 00000000000..7baa4f3169c --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m0.kt @@ -0,0 +1,2 @@ + +fun box0() = qux() diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m1.kt b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m1.kt new file mode 100644 index 00000000000..6ff5fd2197b --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/m1.kt @@ -0,0 +1,2 @@ + +fun box1() = qux() diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/module.info b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/module.info new file mode 100644 index 00000000000..6d880d391ff --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/main/module.info @@ -0,0 +1,9 @@ +STEP 0: + dependencies: stdlib, lib1 + dirty: m.kt, m0.kt, m1.kt + +STEP 1: + dependencies: stdlib, lib1 + modifications: + U : m.kt.1.txt -> m.kt + dirty: m.kt diff --git a/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/project.info b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/project.info new file mode 100644 index 00000000000..3a4c1b58fea --- /dev/null +++ b/js/js.translator/testData/incremental/invalidation/mainModuleInvalidation/project.info @@ -0,0 +1,7 @@ +MODULES: lib1, main + +STEP 0: + libs: lib1, main + +STEP 1: + libs: lib1, main diff --git a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info index afee833fa34..1e581fc29be 100644 --- a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info @@ -5,3 +5,6 @@ STEP 1: dependencies: stdlib modifications: D : l1_1.kt +STEP 2: + dependencies: stdlib + flags: FP diff --git a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/main/module.info b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/main/module.info index 7424db65899..7dcd9e4a2f0 100644 --- a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/main/module.info +++ b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/main/module.info @@ -1,7 +1,8 @@ STEP 0: dependencies: stdlib, lib1 dirty: m.kt - STEP 1: dependencies: stdlib, lib1 - flags: FP \ No newline at end of file +STEP 2: + dependencies: stdlib, lib1 + flags: FP diff --git a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/project.info b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/project.info index e43922d5598..928fb4c8855 100644 --- a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/project.info +++ b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/project.info @@ -4,4 +4,7 @@ STEP 0: libs: lib1, main STEP 1: - libs: lib1, main \ No newline at end of file + libs: lib1, main + +STEP 2: + libs: lib1, main diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt index e064f208a57..5f955e49634 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/targets/js/ir/KotlinJsIrLink.kt @@ -26,9 +26,12 @@ import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.DEVELOPMENT import org.jetbrains.kotlin.gradle.targets.js.dsl.KotlinJsBinaryMode.PRODUCTION import org.jetbrains.kotlin.gradle.tasks.Kotlin2JsCompile import org.jetbrains.kotlin.gradle.utils.getValue +import org.jetbrains.kotlin.gradle.utils.toHexString import org.jetbrains.kotlin.statistics.metrics.BooleanMetrics import org.jetbrains.kotlin.statistics.metrics.StringMetrics import java.io.File +import java.nio.charset.StandardCharsets +import java.security.MessageDigest import javax.inject.Inject @CacheableTask @@ -65,10 +68,6 @@ abstract class KotlinJsIrLink @Inject constructor( @get:Internal internal val propertiesProvider = PropertiesProvider(project) - @get:Inject - open val fileHasher: FileHasher - get() = throw UnsupportedOperationException() - @get:Input internal val incrementalJsIr: Boolean = propertiesProvider.incrementalJsIr @@ -125,15 +124,17 @@ abstract class KotlinJsIrLink @Inject constructor( ) } if (incrementalJsIr && mode == DEVELOPMENT) { + val digest = MessageDigest.getInstance("SHA-256") args.cacheDirectories = args.libraries?.splitByPathSeparator() ?.map { val file = File(it) + val hash = digest.digest(file.normalize().absolutePath.toByteArray(StandardCharsets.UTF_8)).toHexString() rootCacheDirectory .resolve(file.nameWithoutExtension) + .resolve(hash) .also { it.mkdirs() } - .resolve(fileHasher.hash(file).toString()) } ?.plus(rootCacheDirectory.resolve(entryModule.get().asFile.name)) ?.let { diff --git a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt index 86604031a33..8a8fe7850a5 100644 --- a/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt +++ b/libraries/tools/kotlin-gradle-plugin/src/main/kotlin/org/jetbrains/kotlin/gradle/utils/cacheKlibUtils.kt @@ -43,7 +43,7 @@ fun getCacheDirectory( return File(cacheDirectory, computeDependenciesHash(dependency)) } -private fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } +internal fun ByteArray.toHexString() = joinToString("") { (0xFF and it.toInt()).toString(16).padStart(2, '0') } private fun computeDependenciesHash(dependency: ResolvedDependency): String { val allArtifactsPaths = @@ -103,4 +103,4 @@ internal class GradleLoggerAdapter(private val gradleLogger: Logger) : org.jetbr override fun warning(message: String) = gradleLogger.warn(message) override fun error(message: String) = kotlin.error(message) override fun fatal(message: String): Nothing = kotlin.error(message) -} \ No newline at end of file +}