diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index 54bf602122b..3cfef17046b 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -5,15 +5,12 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.config.CompilerConfiguration -import org.jetbrains.kotlin.descriptors.DescriptorVisibilities import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment import org.jetbrains.kotlin.ir.declarations.* -import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.ir.util.* -import org.jetbrains.kotlin.ir.util.resolveFakeOverride import org.jetbrains.kotlin.js.config.JSConfigurationKeys import org.jetbrains.kotlin.konan.properties.propertyList import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS @@ -57,7 +54,7 @@ class CacheUpdater( private val mainArguments: List?, private val compilerInterfaceFactory: JsIrCompilerICInterfaceFactory ) { - private val stopwatch = StopwatchIC().apply { startNext("Cache initializing and klibs loading") } + private val stopwatch = StopwatchIC().apply { startNext("Loading klibs") } private val libraries = loadLibraries(allModules) private val dependencyGraph = buildDependenciesGraph(libraries) @@ -68,11 +65,6 @@ class CacheUpdater( private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) private val mainLibrary = libraries[mainLibraryFile] ?: notFoundIcError("main library", mainLibraryFile) - private val incrementalCaches = libraries.entries.associate { (libFile, lib) -> - val cachePath = cacheMap[lib] ?: notFoundIcError("cache path", KotlinLibraryFile(lib)) - libFile to IncrementalCache(lib, cachePath) - } - private val dirtyFileStats = KotlinSourceFileMutableMap>() init { @@ -90,9 +82,6 @@ class CacheUpdater( } } - private fun getLibIncrementalCache(libFile: KotlinLibraryFile) = - incrementalCaches[libFile] ?: notFoundIcError("incremental cache", libFile) - private fun loadLibraries(allModules: Collection): Map { val allResolvedDependencies = jsResolveLibraries( allModules, @@ -112,6 +101,20 @@ class CacheUpdater( } } + private fun mapLibrariesInOrder(f: (KotlinLibraryFile) -> T): List { + val visited = HashSet(libraries.size) + val result = ArrayList(libraries.size) + fun visitLibrary(lib: KotlinLibrary) { + if (visited.add(lib)) { + dependencyGraph[lib]?.forEach(::visitLibrary) + result += f(KotlinLibraryFile(lib)) + } + } + + visitLibrary(mainLibrary) + return result + } + private class DirtyFileExports( override val inverseDependencies: KotlinSourceFileMutableMap> = KotlinSourceFileMutableMap() ) : KotlinSourceFileExports() { @@ -155,435 +158,446 @@ class CacheUpdater( fun isExportedSignaturesUpdated() = newExportedSignatures != oldInverseDependencies.flatSignatures() } - private fun addFilesWithRemovedDependencies( - modifiedFiles: KotlinSourceFileMap, removedFiles: KotlinSourceFileMap - ): KotlinSourceFileMap { - val extraModifiedLibFiles = KotlinSourceFileMutableMap() - - fun addDependenciesToExtraModifiedFiles(dependencies: KotlinSourceFileMap<*>, dirtyState: DirtyFileState) { - for ((dependentLib, dependentFiles) in dependencies) { - val dependentCache = incrementalCaches[dependentLib] ?: continue - val alreadyModifiedFiles = modifiedFiles[dependentLib] ?: emptyMap() - val alreadyRemovedFiles = removedFiles[dependentLib] ?: emptyMap() - val extraModifiedFiles by lazy(LazyThreadSafetyMode.NONE) { extraModifiedLibFiles.getOrPutFiles(dependentLib) } - val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(dependentLib) } - for (dependentFile in dependentFiles.keys) { - when (dependentFile) { - in alreadyModifiedFiles -> continue - in alreadyRemovedFiles -> continue - in extraModifiedFiles -> continue - else -> { - val dependentMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile) - extraModifiedFiles[dependentFile] = dependentMetadata - fileStats.addDirtFileStat(dependentFile, dirtyState) - } - } - } - } - } - - removedFiles.forEachFile { _, _, removedFileMetadata -> - addDependenciesToExtraModifiedFiles(removedFileMetadata.directDependencies, DirtyFileState.REMOVED_INVERSE_DEPENDS) - addDependenciesToExtraModifiedFiles(removedFileMetadata.inverseDependencies, DirtyFileState.REMOVED_DIRECT_DEPENDS) - } - - if (extraModifiedLibFiles.isNotEmpty()) { - extraModifiedLibFiles.copyFilesFrom(modifiedFiles) - return extraModifiedLibFiles - } - return modifiedFiles - } - - private fun loadModifiedFiles(): KotlinSourceFileMap { - val removedFilesMetadata = hashMapOf>() - - val modifiedFiles = KotlinSourceFileMap(incrementalCaches.entries.associate { (lib, cache) -> - val (dirtyFiles, removedFiles, newFiles, modifiedConfigFiles) = cache.collectModifiedFiles(configHash) - - val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(lib) } - newFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) } - modifiedConfigFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.MODIFIED_CONFIG) } - removedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.REMOVED_FILE) } - dirtyFiles.forEach { - if (it.key !in newFiles && it.key !in modifiedConfigFiles) { - fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR) - } - } - - if (removedFiles.isNotEmpty()) { - removedFilesMetadata[lib] = removedFiles - } - - lib to dirtyFiles - }) - - return addFilesWithRemovedDependencies(modifiedFiles, KotlinSourceFileMap(removedFilesMetadata)) - } - - private fun collectExportedSymbolsForDirtyFiles( - dirtyFiles: KotlinSourceFileMap - ): KotlinSourceFileMutableMap { - val exportedSymbols = KotlinSourceFileMutableMap() - - for ((libFile, srcFiles) in dirtyFiles) { - val exportedSymbolFiles = HashMap(srcFiles.size) - for ((srcFile, srcFileMetadata) in srcFiles) { - val loadingFileExports = DirtyFileExports() - for ((dependentLib, dependentFiles) in srcFileMetadata.inverseDependencies) { - val dependentCache = incrementalCaches[dependentLib] ?: continue - val dirtyLibFiles = dirtyFiles[dependentLib] ?: emptyMap() - for (dependentFile in dependentFiles.keys) { - if (dependentFile !in dirtyLibFiles) { - val dependentSrcFileMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile) - dependentSrcFileMetadata.directDependencies[libFile, srcFile]?.let { - loadingFileExports.inverseDependencies[dependentLib, dependentFile] = it.keys - loadingFileExports.allExportedSignatures += it.keys - } - } - } - } - exportedSymbolFiles[srcFile] = loadingFileExports - } - if (exportedSymbolFiles.isNotEmpty()) { - exportedSymbols[libFile] = exportedSymbolFiles - } - } - return exportedSymbols - } - - private fun KotlinSourceFileMutableMap.getExportedSignaturesAndAddMetadata( - jsIrLinker: JsIrLinker, - irModule: IrModuleFragment, - libFile: KotlinLibraryFile, - dirtySrcFiles: Set - ): Map { - val idSignatureToFile = hashMapOf() - val moduleDeserializer = jsIrLinker.moduleDeserializer(irModule.descriptor) - val incrementalCache = getLibIncrementalCache(libFile) - for (fileDeserializer in moduleDeserializer.fileDeserializers()) { - val reachableSignatures = fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() - val maybeImportedSignatures = HashSet(reachableSignatures.keys) - val implementedSymbols = collectImplementedSymbol(fileDeserializer.symbolDeserializer.deserializedSymbols) - for ((signature, symbol) in implementedSymbols) { - var symbolCanBeExported = maybeImportedSignatures.remove(signature) - resolveFakeOverrideFunction(symbol)?.let { resolvedSignature -> - if (resolvedSignature !in implementedSymbols) { - maybeImportedSignatures.add(resolvedSignature) - } - symbolCanBeExported = true - } - if (symbolCanBeExported) { - idSignatureToFile[signature] = IdSignatureSource(libFile, KotlinSourceFile(fileDeserializer.file), symbol) - } - } - - val libSrcFile = KotlinSourceFile(fileDeserializer.file) - if (libSrcFile in dirtySrcFiles) { - val metadata = incrementalCache.fetchSourceFileFullMetadata(libSrcFile) - this[libFile, libSrcFile] = DirtyFileMetadata(maybeImportedSignatures, metadata.directDependencies) - } - } - return idSignatureToFile - } - - private fun DirtyFileMetadata.setAllDependencies( - signatureHashCalculator: IdSignatureHashCalculator, - idSignatureToFile: Map, - updatedMetadata: KotlinSourceFileMap, - libFile: KotlinLibraryFile, - srcFile: KotlinSourceFile - ) { - val allImportedSignatures = addParentSignatures(maybeImportedSignatures, idSignatureToFile, libFile, srcFile) - for (importedSignature in allImportedSignatures) { - val (dependencyLib, dependencyFile) = idSignatureToFile[importedSignature] ?: continue - signatureHashCalculator[importedSignature]?.also { signatureHash -> - addDirectDependency(dependencyLib, dependencyFile, importedSignature, signatureHash) - } ?: notFoundIcError("signature $importedSignature hash", dependencyLib, dependencyFile) - - updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata -> - dependencyMetadata.addInverseDependency(libFile, srcFile, importedSignature) - } - } - } - - private fun rebuildDirtySourceMetadata( - signatureHashCalculator: IdSignatureHashCalculator, - jsIrLinker: JsIrLinker, - loadedFragments: Map, - dirtySrcFiles: KotlinSourceFileMap, - ): KotlinSourceFileMap { - val idSignatureToFile = hashMapOf() - val updatedMetadata = KotlinSourceFileMutableMap() - - for ((lib, irModule) in loadedFragments) { - val libDirtySrcFiles = dirtySrcFiles[lib]?.keys ?: emptySet() - idSignatureToFile += updatedMetadata.getExportedSignaturesAndAddMetadata(jsIrLinker, irModule, lib, libDirtySrcFiles) - } - - signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile) - - for ((libFile, srcFiles) in updatedMetadata) { - val libDirtySrcFiles = dirtySrcFiles[libFile] ?: continue - for ((srcFile, updatedHeader) in srcFiles) { - val dirtySrcFile = libDirtySrcFiles[srcFile] ?: continue - dirtySrcFile.inverseDependencies.forEachFile { dependentLibFile, dependentSrcFile, signatures -> - signatures.forEach { signature -> - val signatureSrc = idSignatureToFile[signature] - val dependencyLib = signatureSrc?.lib ?: libFile - val dependencyFile = signatureSrc?.src ?: srcFile - updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata -> - dependencyMetadata.addInverseDependency(dependentLibFile, dependentSrcFile, signature) - } - } - } - - updatedHeader.setAllDependencies(signatureHashCalculator, idSignatureToFile, updatedMetadata, libFile, srcFile) - } - } - - val result = KotlinSourceFileMutableMap() - - for ((libFile, sourceFiles) in dirtySrcFiles) { - val incrementalCache = getLibIncrementalCache(libFile) - val srcFileUpdatedMetadata = updatedMetadata[libFile] ?: notFoundIcError("metadata", libFile) - for (srcFile in sourceFiles.keys) { - val srcMetadata = srcFileUpdatedMetadata[srcFile] ?: notFoundIcError("metadata", libFile, srcFile) - incrementalCache.updateSourceFileMetadata(srcFile, srcMetadata) - result[libFile, srcFile] = srcMetadata - } - } - - return result - } - private fun KotlinSourceFileMutableMap.addNewMetadata( libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, oldMetadata: KotlinSourceFileMetadata ) = this[libFile, srcFile] ?: UpdatedDependenciesMetadata(oldMetadata).also { this[libFile, srcFile] = it } - private fun KotlinSourceFileMutableMap.addDependenciesWithUpdatedSignatures( - libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata - ) { - // go through dependencies and collect dependencies with updated signatures - for ((dependencyLibFile, dependencySrcFiles) in srcFileMetadata.directDependencies) { - val dependencyCache = getLibIncrementalCache(dependencyLibFile) - for ((dependencySrcFile, newSignatures) in dependencySrcFiles) { - val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(dependencySrcFile) - val oldSignatures = dependencySrcMetadata.inverseDependencies[libFile, srcFile] ?: emptySet() - if (oldSignatures == newSignatures) { - continue - } - val newMetadata = addNewMetadata(dependencyLibFile, dependencySrcFile, dependencySrcMetadata) - newMetadata.inverseDependencies[libFile, srcFile] = newSignatures.keys - } - } - } + private inner class CacheUpdaterInternal { + val signatureHashCalculator = IdSignatureHashCalculator() - private fun KotlinSourceFileMutableMap.addDependenciesWithRemovedInverseDependencies( - libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata - ) { - // go through old dependencies and look for removed dependencies - for ((oldDependencyLibFile, oldDependencySrcFiles) in srcFileMetadata.oldDirectDependencies) { - val dependencyCache = incrementalCaches[oldDependencyLibFile] ?: continue - val newDirectDependencyFiles = srcFileMetadata.directDependencies[oldDependencyLibFile] ?: emptyMap() - for (oldDependencySrcFile in oldDependencySrcFiles.keys) { - if (oldDependencySrcFile in newDirectDependencyFiles) { - continue + private val incrementalCaches = libraries.entries.associate { (libFile, lib) -> + val cachePath = cacheMap[lib] ?: notFoundIcError("cache path", KotlinLibraryFile(lib)) + libFile to IncrementalCache(lib, cachePath) + } + + private fun getLibIncrementalCache(libFile: KotlinLibraryFile) = + incrementalCaches[libFile] ?: notFoundIcError("incremental cache", libFile) + + private fun addFilesWithRemovedDependencies( + modifiedFiles: KotlinSourceFileMap, removedFiles: KotlinSourceFileMap + ): KotlinSourceFileMap { + val extraModifiedLibFiles = KotlinSourceFileMutableMap() + + fun addDependenciesToExtraModifiedFiles(dependencies: KotlinSourceFileMap<*>, dirtyState: DirtyFileState) { + for ((dependentLib, dependentFiles) in dependencies) { + val dependentCache = incrementalCaches[dependentLib] ?: continue + val alreadyModifiedFiles = modifiedFiles[dependentLib] ?: emptyMap() + val alreadyRemovedFiles = removedFiles[dependentLib] ?: emptyMap() + val extraModifiedFiles by lazy(LazyThreadSafetyMode.NONE) { extraModifiedLibFiles.getOrPutFiles(dependentLib) } + val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(dependentLib) } + for (dependentFile in dependentFiles.keys) { + when (dependentFile) { + in alreadyModifiedFiles -> continue + in alreadyRemovedFiles -> continue + in extraModifiedFiles -> continue + else -> { + val dependentMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile) + extraModifiedFiles[dependentFile] = dependentMetadata + fileStats.addDirtFileStat(dependentFile, dirtyState) + } + } + } } - val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(oldDependencySrcFile) - if (dependencySrcMetadata.inverseDependencies[libFile, srcFile] != null) { - val newMetadata = addNewMetadata(oldDependencyLibFile, oldDependencySrcFile, dependencySrcMetadata) - newMetadata.inverseDependencies.removeFile(libFile, srcFile) + } + + removedFiles.forEachFile { _, _, removedFileMetadata -> + addDependenciesToExtraModifiedFiles(removedFileMetadata.directDependencies, DirtyFileState.REMOVED_INVERSE_DEPENDS) + addDependenciesToExtraModifiedFiles(removedFileMetadata.inverseDependencies, DirtyFileState.REMOVED_DIRECT_DEPENDS) + } + + if (extraModifiedLibFiles.isNotEmpty()) { + extraModifiedLibFiles.copyFilesFrom(modifiedFiles) + return extraModifiedLibFiles + } + return modifiedFiles + } + + fun loadModifiedFiles(): KotlinSourceFileMap { + val removedFilesMetadata = hashMapOf>() + + val modifiedFiles = KotlinSourceFileMap(incrementalCaches.entries.associate { (lib, cache) -> + val (dirtyFiles, removedFiles, newFiles, modifiedConfigFiles) = cache.collectModifiedFiles(configHash) + + val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(lib) } + newFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) } + modifiedConfigFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.MODIFIED_CONFIG) } + removedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.REMOVED_FILE) } + dirtyFiles.forEach { + if (it.key !in newFiles && it.key !in modifiedConfigFiles) { + fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR) + } + } + + if (removedFiles.isNotEmpty()) { + removedFilesMetadata[lib] = removedFiles + } + + lib to dirtyFiles + }) + + return addFilesWithRemovedDependencies(modifiedFiles, KotlinSourceFileMap(removedFilesMetadata)) + } + + fun collectExportedSymbolsForDirtyFiles( + dirtyFiles: KotlinSourceFileMap + ): KotlinSourceFileMutableMap { + val exportedSymbols = KotlinSourceFileMutableMap() + + for ((libFile, srcFiles) in dirtyFiles) { + val exportedSymbolFiles = HashMap(srcFiles.size) + for ((srcFile, srcFileMetadata) in srcFiles) { + val loadingFileExports = DirtyFileExports() + for ((dependentLib, dependentFiles) in srcFileMetadata.inverseDependencies) { + val dependentCache = incrementalCaches[dependentLib] ?: continue + val dirtyLibFiles = dirtyFiles[dependentLib] ?: emptyMap() + for (dependentFile in dependentFiles.keys) { + if (dependentFile !in dirtyLibFiles) { + val dependentSrcFileMetadata = dependentCache.fetchSourceFileFullMetadata(dependentFile) + dependentSrcFileMetadata.directDependencies[libFile, srcFile]?.let { + loadingFileExports.inverseDependencies[dependentLib, dependentFile] = it.keys + loadingFileExports.allExportedSignatures += it.keys + } + } + } + } + exportedSymbolFiles[srcFile] = loadingFileExports + } + if (exportedSymbolFiles.isNotEmpty()) { + exportedSymbols[libFile] = exportedSymbolFiles + } + } + return exportedSymbols + } + + private fun KotlinSourceFileMutableMap.getExportedSignaturesAndAddMetadata( + jsIrLinker: JsIrLinker, + irModule: IrModuleFragment, + libFile: KotlinLibraryFile, + dirtySrcFiles: Set + ): Map { + val idSignatureToFile = hashMapOf() + val moduleDeserializer = jsIrLinker.moduleDeserializer(irModule.descriptor) + val incrementalCache = getLibIncrementalCache(libFile) + for (fileDeserializer in moduleDeserializer.fileDeserializers()) { + val reachableSignatures = fileDeserializer.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() + val maybeImportedSignatures = HashSet(reachableSignatures.keys) + val implementedSymbols = collectImplementedSymbol(fileDeserializer.symbolDeserializer.deserializedSymbols) + for ((signature, symbol) in implementedSymbols) { + var symbolCanBeExported = maybeImportedSignatures.remove(signature) + resolveFakeOverrideFunction(symbol)?.let { resolvedSignature -> + if (resolvedSignature !in implementedSymbols) { + maybeImportedSignatures.add(resolvedSignature) + } + symbolCanBeExported = true + } + if (symbolCanBeExported) { + idSignatureToFile[signature] = IdSignatureSource(libFile, KotlinSourceFile(fileDeserializer.file), symbol) + } + } + + val libSrcFile = KotlinSourceFile(fileDeserializer.file) + if (libSrcFile in dirtySrcFiles) { + val metadata = incrementalCache.fetchSourceFileFullMetadata(libSrcFile) + this[libFile, libSrcFile] = DirtyFileMetadata(maybeImportedSignatures, metadata.directDependencies) + } + } + return idSignatureToFile + } + + private fun DirtyFileMetadata.setAllDependencies( + idSignatureToFile: Map, + updatedMetadata: KotlinSourceFileMap, + libFile: KotlinLibraryFile, + srcFile: KotlinSourceFile + ) { + val allImportedSignatures = addParentSignatures(maybeImportedSignatures, idSignatureToFile, libFile, srcFile) + for (importedSignature in allImportedSignatures) { + val (dependencyLib, dependencyFile) = idSignatureToFile[importedSignature] ?: continue + signatureHashCalculator[importedSignature]?.also { signatureHash -> + addDirectDependency(dependencyLib, dependencyFile, importedSignature, signatureHash) + } ?: notFoundIcError("signature $importedSignature hash", dependencyLib, dependencyFile) + + updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata -> + dependencyMetadata.addInverseDependency(libFile, srcFile, importedSignature) } } } - } - private fun KotlinSourceFileMutableMap.addDependentsWithUpdatedImports( - signatureHashCalculator: IdSignatureHashCalculator, - libFile: KotlinLibraryFile, - srcFile: KotlinSourceFile, - srcFileMetadata: DirtyFileMetadata - ) { - // go through dependent files and check if their imports were modified - for ((dependentLibFile, dependentSrcFiles) in srcFileMetadata.inverseDependencies) { - val dependentCache = incrementalCaches[dependentLibFile] ?: continue - for ((dependentSrcFile, newSignatures) in dependentSrcFiles) { - val dependentSrcMetadata = dependentCache.fetchSourceFileFullMetadata(dependentSrcFile) - val dependentSignatures = dependentSrcMetadata.directDependencies[libFile, srcFile] ?: emptyMap() - when { - dependentSrcMetadata is DirtyFileMetadata -> continue - this[dependentLibFile, dependentSrcFile]?.importedSignaturesModified == true -> continue - dependentSignatures.any { signatureHashCalculator[it.key] != it.value } -> { - val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata) - newMetadata.importedSignaturesModified = true + fun rebuildDirtySourceMetadata( + jsIrLinker: JsIrLinker, + loadedFragments: Map, + dirtySrcFiles: KotlinSourceFileMap, + ): KotlinSourceFileMap { + val idSignatureToFile = hashMapOf() + val updatedMetadata = KotlinSourceFileMutableMap() + + for ((lib, irModule) in loadedFragments) { + val libDirtySrcFiles = dirtySrcFiles[lib]?.keys ?: emptySet() + idSignatureToFile += updatedMetadata.getExportedSignaturesAndAddMetadata(jsIrLinker, irModule, lib, libDirtySrcFiles) + } + + signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile) + + for ((libFile, srcFiles) in updatedMetadata) { + val libDirtySrcFiles = dirtySrcFiles[libFile] ?: continue + for ((srcFile, updatedHeader) in srcFiles) { + val dirtySrcFile = libDirtySrcFiles[srcFile] ?: continue + dirtySrcFile.inverseDependencies.forEachFile { dependentLibFile, dependentSrcFile, signatures -> + signatures.forEach { signature -> + val signatureSrc = idSignatureToFile[signature] + val dependencyLib = signatureSrc?.lib ?: libFile + val dependencyFile = signatureSrc?.src ?: srcFile + updatedMetadata[dependencyLib, dependencyFile]?.also { dependencyMetadata -> + dependencyMetadata.addInverseDependency(dependentLibFile, dependentSrcFile, signature) + } + } } - dependentSignatures.keys != newSignatures -> { - val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata) - newMetadata.directDependencies[libFile, srcFile] = newSignatures.associateWithTo(HashMap(newSignatures.size)) { - signatureHashCalculator[it] ?: notFoundIcError("signature $it hash", libFile, srcFile) + updatedHeader.setAllDependencies(idSignatureToFile, updatedMetadata, libFile, srcFile) + } + } + + val result = KotlinSourceFileMutableMap() + + for ((libFile, sourceFiles) in dirtySrcFiles) { + val incrementalCache = getLibIncrementalCache(libFile) + val srcFileUpdatedMetadata = updatedMetadata[libFile] ?: notFoundIcError("metadata", libFile) + for (srcFile in sourceFiles.keys) { + val srcMetadata = srcFileUpdatedMetadata[srcFile] ?: notFoundIcError("metadata", libFile, srcFile) + incrementalCache.updateSourceFileMetadata(srcFile, srcMetadata) + result[libFile, srcFile] = srcMetadata + } + } + + return result + } + + private fun KotlinSourceFileMutableMap.addDependenciesWithUpdatedSignatures( + libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata + ) { + // go through dependencies and collect dependencies with updated signatures + for ((dependencyLibFile, dependencySrcFiles) in srcFileMetadata.directDependencies) { + val dependencyCache = getLibIncrementalCache(dependencyLibFile) + for ((dependencySrcFile, newSignatures) in dependencySrcFiles) { + val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(dependencySrcFile) + val oldSignatures = dependencySrcMetadata.inverseDependencies[libFile, srcFile] ?: emptySet() + if (oldSignatures == newSignatures) { + continue + } + val newMetadata = addNewMetadata(dependencyLibFile, dependencySrcFile, dependencySrcMetadata) + newMetadata.inverseDependencies[libFile, srcFile] = newSignatures.keys + } + } + } + + private fun KotlinSourceFileMutableMap.addDependenciesWithRemovedInverseDependencies( + libFile: KotlinLibraryFile, srcFile: KotlinSourceFile, srcFileMetadata: DirtyFileMetadata + ) { + // go through old dependencies and look for removed dependencies + for ((oldDependencyLibFile, oldDependencySrcFiles) in srcFileMetadata.oldDirectDependencies) { + val dependencyCache = incrementalCaches[oldDependencyLibFile] ?: continue + val newDirectDependencyFiles = srcFileMetadata.directDependencies[oldDependencyLibFile] ?: emptyMap() + for (oldDependencySrcFile in oldDependencySrcFiles.keys) { + if (oldDependencySrcFile in newDirectDependencyFiles) { + continue + } + val dependencySrcMetadata = dependencyCache.fetchSourceFileFullMetadata(oldDependencySrcFile) + if (dependencySrcMetadata.inverseDependencies[libFile, srcFile] != null) { + val newMetadata = addNewMetadata(oldDependencyLibFile, oldDependencySrcFile, dependencySrcMetadata) + newMetadata.inverseDependencies.removeFile(libFile, srcFile) + } + } + } + } + + private fun KotlinSourceFileMutableMap.addDependentsWithUpdatedImports( + libFile: KotlinLibraryFile, + srcFile: KotlinSourceFile, + srcFileMetadata: DirtyFileMetadata + ) { + // go through dependent files and check if their imports were modified + for ((dependentLibFile, dependentSrcFiles) in srcFileMetadata.inverseDependencies) { + val dependentCache = incrementalCaches[dependentLibFile] ?: continue + for ((dependentSrcFile, newSignatures) in dependentSrcFiles) { + val dependentSrcMetadata = dependentCache.fetchSourceFileFullMetadata(dependentSrcFile) + val dependentSignatures = dependentSrcMetadata.directDependencies[libFile, srcFile] ?: emptyMap() + when { + dependentSrcMetadata is DirtyFileMetadata -> continue + this[dependentLibFile, dependentSrcFile]?.importedSignaturesModified == true -> continue + dependentSignatures.any { signatureHashCalculator[it.key] != it.value } -> { + val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata) + newMetadata.importedSignaturesModified = true + } + + dependentSignatures.keys != newSignatures -> { + val newMetadata = addNewMetadata(dependentLibFile, dependentSrcFile, dependentSrcMetadata) + newMetadata.directDependencies[libFile, srcFile] = newSignatures.associateWithTo(HashMap(newSignatures.size)) { + signatureHashCalculator[it] ?: notFoundIcError("signature $it hash", libFile, srcFile) + } } } } } } - } - private fun collectFilesWithModifiedExportsAndImports( - signatureHashCalculator: IdSignatureHashCalculator, - loadedDirtyFiles: KotlinSourceFileMap - ): KotlinSourceFileMap { - val filesWithModifiedExportsAndImports = KotlinSourceFileMutableMap() + fun collectFilesWithModifiedExportsAndImports( + loadedDirtyFiles: KotlinSourceFileMap + ): KotlinSourceFileMap { + val filesWithModifiedExportsAndImports = KotlinSourceFileMutableMap() - loadedDirtyFiles.forEachFile { libFile, srcFile, srcFileMetadata -> - filesWithModifiedExportsAndImports.addDependenciesWithUpdatedSignatures(libFile, srcFile, srcFileMetadata) - filesWithModifiedExportsAndImports.addDependenciesWithRemovedInverseDependencies(libFile, srcFile, srcFileMetadata) - filesWithModifiedExportsAndImports.addDependentsWithUpdatedImports(signatureHashCalculator, libFile, srcFile, srcFileMetadata) + loadedDirtyFiles.forEachFile { libFile, srcFile, srcFileMetadata -> + filesWithModifiedExportsAndImports.addDependenciesWithUpdatedSignatures(libFile, srcFile, srcFileMetadata) + filesWithModifiedExportsAndImports.addDependenciesWithRemovedInverseDependencies(libFile, srcFile, srcFileMetadata) + filesWithModifiedExportsAndImports.addDependentsWithUpdatedImports(libFile, srcFile, srcFileMetadata) + } + + return filesWithModifiedExportsAndImports } - return filesWithModifiedExportsAndImports - } + fun collectFilesToRebuildSignatures( + filesWithModifiedExports: KotlinSourceFileMap + ): KotlinSourceFileMap { + val libFilesToRebuild = KotlinSourceFileMutableMap() - private fun collectFilesToRebuildSignatures( - filesWithModifiedExports: KotlinSourceFileMap - ): KotlinSourceFileMap { - val libFilesToRebuild = KotlinSourceFileMutableMap() + for ((libFile, srcFiles) in filesWithModifiedExports) { + val filesToRebuild by lazy(LazyThreadSafetyMode.NONE) { libFilesToRebuild.getOrPutFiles(libFile) } + val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(libFile) } + val cache = getLibIncrementalCache(libFile) - for ((libFile, srcFiles) in filesWithModifiedExports) { - val filesToRebuild by lazy(LazyThreadSafetyMode.NONE) { libFilesToRebuild.getOrPutFiles(libFile) } - val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(libFile) } - val cache = getLibIncrementalCache(libFile) - - for ((srcFile, srcFileMetadata) in srcFiles) { - val isExportedSignatureUpdated = srcFileMetadata.isExportedSignaturesUpdated() - if (isExportedSignatureUpdated || srcFileMetadata.importedSignaturesModified) { - // if exported signatures or imported inline functions were modified - rebuild - filesToRebuild[srcFile] = srcFileMetadata - if (isExportedSignatureUpdated) { - fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_EXPORTS) + for ((srcFile, srcFileMetadata) in srcFiles) { + val isExportedSignatureUpdated = srcFileMetadata.isExportedSignaturesUpdated() + if (isExportedSignatureUpdated || srcFileMetadata.importedSignaturesModified) { + // if exported signatures or imported inline functions were modified - rebuild + filesToRebuild[srcFile] = srcFileMetadata + if (isExportedSignatureUpdated) { + fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_EXPORTS) + } + if (srcFileMetadata.importedSignaturesModified) { + fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_IMPORTS) + } + } else { + // if signatures and inline functions are the same - just update cache metadata + cache.updateSourceFileMetadata(srcFile, srcFileMetadata) } - if (srcFileMetadata.importedSignaturesModified) { - fileStats.addDirtFileStat(srcFile, DirtyFileState.UPDATED_IMPORTS) + } + } + + return libFilesToRebuild + } + + fun updateStdlibIntrinsicDependencies( + linker: JsIrLinker, + mainModule: IrModuleFragment, + loadedFragments: Map, + dirtyFiles: Map> + ) { + val stdlibDescriptor = mainModule.descriptor.builtIns.builtInsModule + val (stdlibFile, stdlibIr) = loadedFragments.entries.find { + it.value.descriptor === stdlibDescriptor + } ?: notFoundIcError("stdlib loaded fragment") + + val stdlibDirtyFiles = dirtyFiles[stdlibFile] ?: return + + signatureHashCalculator.updateInlineFunctionTransitiveHashes(listOf(stdlibIr)) + + val updatedMetadata = KotlinSourceFileMutableMap() + val idSignatureToFile = updatedMetadata.getExportedSignaturesAndAddMetadata(linker, stdlibIr, stdlibFile, stdlibDirtyFiles) + + signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile) + + updatedMetadata.forEachFile { libFile, srcFile, updatedHeader -> + updatedHeader.setAllDependencies(idSignatureToFile, updatedMetadata, libFile, srcFile) + } + + val incrementalCache = getLibIncrementalCache(stdlibFile) + updatedMetadata.forEachFile { libFile, srcFile, updatedHeader -> + if (libFile != stdlibFile) { + icError("unexpected lib while parsing stdlib dependencies", libFile, srcFile) + } + + val cachedHeader = incrementalCache.fetchSourceFileFullMetadata(srcFile) + + val needUpdate = when { + !updatedHeader.directDependencies.allFiles { lib, file, dependencies -> + cachedHeader.directDependencies[lib, file]?.keys?.containsAll(dependencies.keys) ?: dependencies.isEmpty() + } -> true + + !updatedHeader.inverseDependencies.allFiles { lib, file, invDependencies -> + cachedHeader.inverseDependencies[lib, file]?.containsAll(invDependencies) ?: invDependencies.isEmpty() + } -> true + + else -> false + } + if (needUpdate) { + cachedHeader.directDependencies.forEachFile { lib, file, dependencies -> + val updatedDependencies = updatedHeader.directDependencies[lib, file] + if (updatedDependencies != null) { + updatedDependencies += dependencies + } else { + updatedHeader.directDependencies[lib, file] = HashMap(dependencies) + } } - } else { - // if signatures and inline functions are the same - just update cache metadata - cache.updateSourceFileMetadata(srcFile, srcFileMetadata) + cachedHeader.inverseDependencies.forEachFile { lib, file, dependencies -> + val updatedDependencies = updatedHeader.inverseDependencies[lib, file] + if (updatedDependencies != null) { + updatedDependencies += dependencies + } else { + updatedHeader.inverseDependencies[lib, file] = HashSet(dependencies) + } + } + incrementalCache.updateSourceFileMetadata(srcFile, updatedHeader) } } } - return libFilesToRebuild + fun buildCacheArtifacts( + jsIrLinker: JsIrLinker, + loadedFragments: Map + ): Map { + return mapLibrariesInOrder { libFile -> + val incrementalCache = getLibIncrementalCache(libFile) + val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile) + val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor) + + val signatureToIndexMapping = moduleDeserializer.fileDeserializers().associate { + KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() + } + libFile to incrementalCache.buildIncrementalCacheArtifact(signatureToIndexMapping) + }.toMap() + } } - private fun buildModuleArtifactsAndCommitCache( - jsIrLinker: JsIrLinker, + private fun commitCacheAndBuildModuleArtifacts( + incrementalCacheArtifacts: Map, loadedFragments: Map, rebuiltFileFragments: List> ): List { - stopwatch.startNext("Cache committing") + stopwatch.startNext("Incremental cache - committing artifacts") val fragmentToLibName = loadedFragments.entries.associate { it.value to it.key } val rebuiltSrcFiles = rebuiltFileFragments.groupBy { fragmentToLibName[it.first.module] ?: notFoundIcError("loaded fragment lib name", srcFile = KotlinSourceFile(it.first)) } - val visited = hashSetOf() - val artifacts = mutableListOf() - fun addArtifact(lib: KotlinLibrary) { - if (visited.add(lib)) { - dependencyGraph[lib]?.forEach(::addArtifact) + val artifacts = mapLibrariesInOrder { libFile -> + val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile) + val libRebuiltFiles = rebuiltSrcFiles[libFile]?.associate { KotlinSourceFile(it.first) to it.second } ?: emptyMap() + val lib = libraries[libFile] ?: notFoundIcError("library", libFile) + val incrementalCacheArtifact = incrementalCacheArtifacts[libFile] ?: notFoundIcError("cache artifact", libFile) - val libFile = KotlinLibraryFile(lib) - val incrementalCache = getLibIncrementalCache(libFile) - val libFragment = loadedFragments[libFile] ?: notFoundIcError("loaded fragment", libFile) - val libRebuiltFiles = rebuiltSrcFiles[libFile]?.associate { KotlinSourceFile(it.first) to it.second } ?: emptyMap() - val moduleDeserializer = jsIrLinker.moduleDeserializer(libFragment.descriptor) - - val signatureToIndexMapping = moduleDeserializer.fileDeserializers().associate { - KotlinSourceFile(it.file) to it.symbolDeserializer.signatureDeserializer.signatureToIndexMapping() - } - artifacts += incrementalCache.buildModuleArtifactAndCommitCache( - moduleName = libFragment.name.asString(), - externalModuleName = lib.jsOutputName, - rebuiltFileFragments = libRebuiltFiles, - signatureToIndexMapping = signatureToIndexMapping - ) - } + incrementalCacheArtifact.buildModuleArtifactAndCommitCache( + moduleName = libFragment.name.asString(), + externalModuleName = lib.jsOutputName, + rebuiltFileFragments = libRebuiltFiles, + ) } - addArtifact(mainLibrary) stopwatch.stop() return artifacts } - private fun updateStdlibIntrinsicDependencies( - signatureHashCalculator: IdSignatureHashCalculator, - linker: JsIrLinker, - mainModule: IrModuleFragment, - loadedFragments: Map, - dirtyFiles: Map> - ) { - val stdlibDescriptor = mainModule.descriptor.builtIns.builtInsModule - val (stdlibFile, stdlibIr) = loadedFragments.entries.find { - it.value.descriptor === stdlibDescriptor - } ?: notFoundIcError("stdlib loaded fragment") - - val stdlibDirtyFiles = dirtyFiles[stdlibFile] ?: return - - signatureHashCalculator.updateInlineFunctionTransitiveHashes(listOf(stdlibIr)) - - val updatedMetadata = KotlinSourceFileMutableMap() - val idSignatureToFile = updatedMetadata.getExportedSignaturesAndAddMetadata(linker, stdlibIr, stdlibFile, stdlibDirtyFiles) - - signatureHashCalculator.addAllSignatureSymbols(idSignatureToFile) - - updatedMetadata.forEachFile { libFile, srcFile, updatedHeader -> - updatedHeader.setAllDependencies(signatureHashCalculator, idSignatureToFile, updatedMetadata, libFile, srcFile) - } - - val incrementalCache = getLibIncrementalCache(stdlibFile) - updatedMetadata.forEachFile { libFile, srcFile, updatedHeader -> - if (libFile != stdlibFile) { - icError("unexpected lib while parsing stdlib dependencies", libFile, srcFile) - } - - val cachedHeader = incrementalCache.fetchSourceFileFullMetadata(srcFile) - - val needUpdate = when { - !updatedHeader.directDependencies.allFiles { lib, file, dependencies -> - cachedHeader.directDependencies[lib, file]?.keys?.containsAll(dependencies.keys) ?: dependencies.isEmpty() - } -> true - - !updatedHeader.inverseDependencies.allFiles { lib, file, invDependencies -> - cachedHeader.inverseDependencies[lib, file]?.containsAll(invDependencies) ?: invDependencies.isEmpty() - } -> true - - else -> false - } - if (needUpdate) { - cachedHeader.directDependencies.forEachFile { lib, file, dependencies -> - val updatedDependencies = updatedHeader.directDependencies[lib, file] - if (updatedDependencies != null) { - updatedDependencies += dependencies - } else { - updatedHeader.directDependencies[lib, file] = HashMap(dependencies) - } - } - cachedHeader.inverseDependencies.forEachFile { lib, file, dependencies -> - val updatedDependencies = updatedHeader.inverseDependencies[lib, file] - if (updatedDependencies != null) { - updatedDependencies += dependencies - } else { - updatedHeader.inverseDependencies[lib, file] = HashSet(dependencies) - } - } - incrementalCache.updateSourceFileMetadata(srcFile, updatedHeader) - } - } - } - private fun compileDirtyFiles( compilerForIC: JsIrCompilerICInterface, loadedFragments: Map, @@ -604,7 +618,7 @@ class CacheUpdater( } private data class IrForDirtyFilesAndCompiler( - val linker: JsIrLinker, + val incrementalCacheArtifacts: Map, val loadedFragments: Map, val dirtyFiles: Map>, val irCompiler: JsIrCompilerICInterface @@ -613,10 +627,14 @@ class CacheUpdater( private fun loadIrForDirtyFilesAndInitCompiler(): IrForDirtyFilesAndCompiler { dirtyFileStats.clear() + stopwatch.startNext("Incremental cache - initializing") + val updater = CacheUpdaterInternal() + stopwatch.startNext("Modified files - checking hashes and collecting") - val modifiedFiles = loadModifiedFiles() + val modifiedFiles = updater.loadModifiedFiles() + stopwatch.startNext("Modified files - collecting exported signatures") - val dirtyFileExports = collectExportedSymbolsForDirtyFiles(modifiedFiles) + val dirtyFileExports = updater.collectExportedSymbolsForDirtyFiles(modifiedFiles) stopwatch.startNext("Modified files - loading and linking IR") val jsIrLinkerLoader = JsIrLinkerLoader(compilerConfiguration, mainLibrary, dependencyGraph, irFactory()) @@ -625,20 +643,18 @@ class CacheUpdater( var iterations = 0 var lastDirtyFiles: KotlinSourceFileMap = dirtyFileExports - val hashCalculator = IdSignatureHashCalculator() - while (true) { stopwatch.startNext("Dependencies ($iterations) - calculating transitive hashes for inline functions") - hashCalculator.updateInlineFunctionTransitiveHashes(loadedIr.loadedFragments.values) + updater.signatureHashCalculator.updateInlineFunctionTransitiveHashes(loadedIr.loadedFragments.values) stopwatch.startNext("Dependencies ($iterations) - updating a dependency graph") - val dirtyMetadata = rebuildDirtySourceMetadata(hashCalculator, loadedIr.linker, loadedIr.loadedFragments, lastDirtyFiles) + val dirtyMetadata = updater.rebuildDirtySourceMetadata(loadedIr.linker, loadedIr.loadedFragments, lastDirtyFiles) stopwatch.startNext("Dependencies ($iterations) - collecting files with updated exports and imports") - val filesWithModifiedExportsOrImports = collectFilesWithModifiedExportsAndImports(hashCalculator, dirtyMetadata) + val filesWithModifiedExportsOrImports = updater.collectFilesWithModifiedExportsAndImports(dirtyMetadata) stopwatch.startNext("Dependencies ($iterations) - collecting exported signatures for files with updated exports and imports") - val filesToRebuild = collectFilesToRebuildSignatures(filesWithModifiedExportsOrImports) + val filesToRebuild = updater.collectFilesToRebuildSignatures(filesWithModifiedExportsOrImports) if (filesToRebuild.isEmpty()) { break @@ -668,18 +684,21 @@ class CacheUpdater( val dirtyFiles = dirtyFileExports.entries.associateTo(HashMap(dirtyFileExports.size)) { it.key to HashSet(it.value.keys) } stopwatch.startNext("Processing IR - updating intrinsics and builtins dependencies") - updateStdlibIntrinsicDependencies(hashCalculator, loadedIr.linker, mainModule, loadedIr.loadedFragments, dirtyFiles) + updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModule, loadedIr.loadedFragments, dirtyFiles) + + stopwatch.startNext("Incremental cache - building artifacts") + val incrementalCachesArtifacts = updater.buildCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments) stopwatch.stop() - return IrForDirtyFilesAndCompiler(loadedIr.linker, loadedIr.loadedFragments, dirtyFiles, compilerForIC) + return IrForDirtyFilesAndCompiler(incrementalCachesArtifacts, loadedIr.loadedFragments, dirtyFiles, compilerForIC) } fun actualizeCaches(): List { - val (irLinker, irFragments, dirtyFiles, irCompiler) = loadIrForDirtyFilesAndInitCompiler() + val (incrementalCachesArtifacts, irFragments, dirtyFiles, irCompiler) = loadIrForDirtyFilesAndInitCompiler() val rebuiltFragments = compileDirtyFiles(irCompiler, irFragments, dirtyFiles) - return buildModuleArtifactsAndCommitCache(irLinker, irFragments, rebuiltFragments) + return commitCacheAndBuildModuleArtifacts(incrementalCachesArtifacts, irFragments, rebuiltFragments) } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index 0a1eb8f6a35..da8eba6e275 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -6,23 +6,21 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer -import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment -import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.impl.javaFile import org.jetbrains.kotlin.protobuf.CodedInputStream import org.jetbrains.kotlin.protobuf.CodedOutputStream -import java.io.BufferedOutputStream import java.io.File -class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { +internal class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { companion object { private const val CACHE_HEADER = "ic.header.bin" private const val BINARY_AST_SUFFIX = "ast.bin" private const val METADATA_SUFFIX = "metadata.bin" + private const val METADATA_TMP_SUFFIX = "metadata.tmp.bin" } private var forceRebuildJs = false @@ -84,29 +82,11 @@ class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { } } - fun buildModuleArtifactAndCommitCache( - moduleName: String, - externalModuleName: String?, - rebuiltFileFragments: Map, - signatureToIndexMapping: Map> - ): ModuleArtifact { - val serializer = JsIrAstSerializer() - + fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map>): IncrementalCacheArtifact { val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile -> - val binaryAstFile = srcFile.getCacheFile(BINARY_AST_SUFFIX) - val rebuiltFileFragment = rebuiltFileFragments[srcFile] - if (rebuiltFileFragment != null) { - binaryAstFile.recreate() - BufferedOutputStream(binaryAstFile.outputStream()).use { bufferedOutStream -> - serializer.serialize(rebuiltFileFragment, bufferedOutStream) - } - } - - commitSourceFileMetadata(srcFile, signatureToIndexMapping[srcFile] ?: emptyMap()) - SrcFileArtifact(srcFile.path, rebuiltFileFragment, binaryAstFile) + commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap()) } - - return ModuleArtifact(moduleName, fileArtifacts, cacheDir, forceRebuildJs, externalModuleName) + return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts) } data class ModifiedFiles( @@ -125,6 +105,7 @@ class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { isConfigModified = cacheHeader.configHash != ICHash() CacheHeader(klibFileHash, configHash) } + cacheHeader.klibFileHash != klibFileHash -> CacheHeader(klibFileHash, configHash) else -> return ModifiedFiles() } @@ -228,15 +209,19 @@ class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { } ?: KotlinSourceFileMetadataNotExist } - private fun commitSourceFileMetadata(srcFile: KotlinSourceFile, signatureToIndexMapping: Map) { + private fun commitSourceFileMetadata( + binaryAstFile: File, + srcFile: KotlinSourceFile, + signatureToIndexMapping: Map + ): SourceFileCacheArtifact { val headerCacheFile = srcFile.getCacheFile(METADATA_SUFFIX) - val sourceFileMetadata = kotlinLibrarySourceFileMetadata[srcFile] ?: return + val sourceFileMetadata = kotlinLibrarySourceFileMetadata[srcFile] + ?: return SourceFileCacheArtifact.DoNotChangeMetadata(srcFile, binaryAstFile) if (sourceFileMetadata.isEmpty()) { - headerCacheFile.delete() - return + return SourceFileCacheArtifact.RemoveMetadata(srcFile, binaryAstFile, headerCacheFile) } if (sourceFileMetadata is KotlinSourceFileMetadataFromDisk) { - return + return SourceFileCacheArtifact.DoNotChangeMetadata(srcFile, binaryAstFile) } val signatureToIndexMappingSaved = signatureToIndexMappingFromMetadata[srcFile] ?: emptyMap() @@ -270,9 +255,11 @@ class IncrementalCache(private val library: KotlinLibrary, cachePath: String) { } } - headerCacheFile.useCodedOutput { + val tmpCacheFile = srcFile.getCacheFile(METADATA_TMP_SUFFIX) + tmpCacheFile.useCodedOutput { writeDirectDependencies(sourceFileMetadata.directDependencies) writeInverseDependencies(sourceFileMetadata.inverseDependencies) } + return SourceFileCacheArtifact.CommitMetadata(srcFile, binaryAstFile, headerCacheFile, tmpCacheFile) } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt new file mode 100644 index 00000000000..36ba4bf3e20 --- /dev/null +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCacheArtifact.kt @@ -0,0 +1,77 @@ +/* + * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.js.ic + +import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.JsIrProgramFragment +import org.jetbrains.kotlin.ir.backend.js.utils.serialization.JsIrAstSerializer +import java.io.BufferedOutputStream +import java.io.File + +internal sealed class SourceFileCacheArtifact(val srcFile: KotlinSourceFile, val binaryAstFile: File) { + abstract fun commitMetadata() + + fun commitBinaryAst(fragment: JsIrProgramFragment, serializer: JsIrAstSerializer) { + binaryAstFile.recreate() + BufferedOutputStream(binaryAstFile.outputStream()).use { bufferedOutStream -> + serializer.serialize(fragment, bufferedOutStream) + } + } + + class DoNotChangeMetadata(srcFile: KotlinSourceFile, binaryAstFile: File) : SourceFileCacheArtifact(srcFile, binaryAstFile) { + override fun commitMetadata() {} + } + + class CommitMetadata( + srcFile: KotlinSourceFile, + binaryAstFile: File, + private val metadataFile: File, + private val tmpMetadataFile: File + ) : SourceFileCacheArtifact(srcFile, binaryAstFile) { + override fun commitMetadata() { + metadataFile.delete() + if (!tmpMetadataFile.renameTo(metadataFile)) { + tmpMetadataFile.copyTo(metadataFile, true) + tmpMetadataFile.delete() + } + + } + } + + class RemoveMetadata( + srcFile: KotlinSourceFile, + binaryAstFile: File, + private val metadataFile: File + ) : SourceFileCacheArtifact(srcFile, binaryAstFile) { + override fun commitMetadata() { + metadataFile.delete() + } + } +} + +internal class IncrementalCacheArtifact( + private val artifactsDir: File, + private val forceRebuildJs: Boolean, + private val srcCacheActions: List +) { + fun buildModuleArtifactAndCommitCache( + moduleName: String, + externalModuleName: String?, + rebuiltFileFragments: Map, + ): ModuleArtifact { + val serializer = JsIrAstSerializer() + + val fileArtifacts = srcCacheActions.map { srcFileAction -> + val rebuiltFileFragment = rebuiltFileFragments[srcFileAction.srcFile] + if (rebuiltFileFragment != null) { + srcFileAction.commitBinaryAst(rebuiltFileFragment, serializer) + } + srcFileAction.commitMetadata() + SrcFileArtifact(srcFileAction.srcFile.path, rebuiltFileFragment, srcFileAction.binaryAstFile) + } + + return ModuleArtifact(moduleName, fileArtifacts, artifactsDir, forceRebuildJs, externalModuleName) + } +}