[JS IR] Invalidate all klib dependencies after removing it

Without the invalidation, broken JS code
 (with broken cross-module references) may appear.

 ^KT-54911 Fixed
This commit is contained in:
Alexander Korepanov
2022-11-21 15:50:52 +01:00
committed by Space Team
parent ca19d71a00
commit 693258ae91
41 changed files with 402 additions and 150 deletions
@@ -18,6 +18,7 @@ import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.resolver.TopologicalLibraryOrder
import org.jetbrains.kotlin.name.FqName
import java.io.File
import java.nio.file.Files
import java.util.EnumSet
fun interface JsIrCompilerICInterface {
@@ -38,6 +39,7 @@ fun interface JsIrCompilerICInterfaceFactory {
enum class DirtyFileState(val str: String) {
ADDED_FILE("added file"),
MODIFIED_IR("modified ir"),
NON_MODIFIED_IR("non modified ir"),
UPDATED_EXPORTS("updated exports"),
UPDATED_IMPORTS("updated imports"),
REMOVED_INVERSE_DEPENDS("removed inverse depends"),
@@ -106,14 +108,25 @@ class CacheUpdater(
private val incrementalCaches = libraryDependencies.keys.associate { lib ->
val libFile = KotlinLibraryFile(lib)
val file = File(libFile.path)
libFile to IncrementalCache(lib, File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}"))
val libraryCacheDir = File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}")
libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
}
private val removedIncrementalCaches = buildList {
if (cacheRootDir.isDirectory) {
val availableCaches = incrementalCaches.values.mapTo(HashSet(incrementalCaches.size)) { it.cacheDir }
val allDirs = Files.walk(cacheRootDir.toPath(), 1).map { it.toFile() }
allDirs.filter { it != cacheRootDir && it !in availableCaches }.forEach { removedCacheDir ->
add(IncrementalCache(KotlinRemovedLibraryHeader(removedCacheDir), removedCacheDir))
}
}
}
private fun getLibIncrementalCache(libFile: KotlinLibraryFile) =
incrementalCaches[libFile] ?: notFoundIcError("incremental cache", libFile)
private fun addFilesWithRemovedDependencies(
modifiedFiles: KotlinSourceFileMap<KotlinSourceFileMetadata>,
modifiedFiles: KotlinSourceFileMutableMap<KotlinSourceFileMetadata>,
removedFiles: KotlinSourceFileMap<KotlinSourceFileMetadata>
): KotlinSourceFileMap<KotlinSourceFileMetadata> {
val extraModifiedLibFiles = KotlinSourceFileMutableMap<KotlinSourceFileMetadata>()
@@ -145,36 +158,42 @@ class CacheUpdater(
addDependenciesToExtraModifiedFiles(removedFileMetadata.inverseDependencies, DirtyFileState.REMOVED_DIRECT_DEPENDS)
}
if (extraModifiedLibFiles.isNotEmpty()) {
extraModifiedLibFiles.copyFilesFrom(modifiedFiles)
return extraModifiedLibFiles
}
modifiedFiles.copyFilesFrom(extraModifiedLibFiles)
return modifiedFiles
}
fun loadModifiedFiles(): KotlinSourceFileMap<KotlinSourceFileMetadata> {
val removedFilesMetadata = hashMapOf<KotlinLibraryFile, Map<KotlinSourceFile, KotlinSourceFileMetadata>>()
val modifiedFiles = KotlinSourceFileMap(incrementalCaches.entries.associate { (lib, cache) ->
val (dirtyFiles, removedFiles, newFiles) = cache.collectModifiedFiles()
fun collectDirtyFiles(lib: KotlinLibraryFile, cache: IncrementalCache): MutableMap<KotlinSourceFile, KotlinSourceFileMetadata> {
val (addedFiles, removedFiles, modifiedFiles, nonModifiedFiles) = cache.collectModifiedFiles()
val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(lib) }
newFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) }
addedFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) }
removedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.REMOVED_FILE) }
dirtyFiles.forEach {
if (it.key !in newFiles) {
fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR)
}
}
modifiedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR) }
nonModifiedFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.NON_MODIFIED_IR) }
if (removedFiles.isNotEmpty()) {
removedFilesMetadata[lib] = removedFiles
}
lib to dirtyFiles
})
return addedFiles.associateWithTo(modifiedFiles.toMutableMap()) { KotlinSourceFileMetadataNotExist }
}
return addFilesWithRemovedDependencies(modifiedFiles, KotlinSourceFileMap(removedFilesMetadata))
for (cache in removedIncrementalCaches) {
val libFile = cache.libraryFileFromHeader ?: notFoundIcError("removed library name; cache dir: ${cache.cacheDir}")
val dirtyFiles = collectDirtyFiles(libFile, cache)
if (dirtyFiles.isNotEmpty()) {
icError("unexpected dirty file", libFile, dirtyFiles.keys.first())
}
}
val dirtyFiles = incrementalCaches.entries.associateTo(hashMapOf()) { (lib, cache) ->
lib to collectDirtyFiles(lib, cache)
}
return addFilesWithRemovedDependencies(KotlinSourceFileMutableMap(dirtyFiles), KotlinSourceFileMap(removedFilesMetadata))
}
fun collectExportedSymbolsForDirtyFiles(
@@ -493,10 +512,15 @@ class CacheUpdater(
}
}
fun buildCacheArtifacts(
fun buildAndCommitCacheArtifacts(
jsIrLinker: JsIrLinker,
loadedFragments: Map<KotlinLibraryFile, IrModuleFragment>
): Map<KotlinLibraryFile, IncrementalCacheArtifact> {
removedIncrementalCaches.forEach {
if (!it.cacheDir.deleteRecursively()) {
icError("can not delete cache directory ${it.cacheDir.absolutePath}")
}
}
return libraryDependencies.keys.associate { library ->
val libFile = KotlinLibraryFile(library)
val incrementalCache = getLibIncrementalCache(libFile)
@@ -616,7 +640,7 @@ class CacheUpdater(
updater.updateStdlibIntrinsicDependencies(loadedIr.linker, mainModuleFragment, loadedIr.loadedFragments, dirtyFiles)
stopwatch.startNext("Incremental cache - building artifacts")
val incrementalCachesArtifacts = updater.buildCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments)
val incrementalCachesArtifacts = updater.buildAndCommitCacheArtifacts(loadedIr.linker, loadedIr.loadedFragments)
stopwatch.stop()
return IrForDirtyFilesAndCompiler(incrementalCachesArtifacts, loadedIr.loadedFragments, dirtyFiles, compilerForIC)
@@ -676,7 +700,7 @@ fun rebuildCacheForDirtyFiles(
}
val libFile = KotlinLibraryFile(library)
val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLibraryHeader(library).sourceFiles
val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLoadedLibraryHeader(library).sourceFileFingerprints.keys
val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })
val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, irFactory)
@@ -48,10 +48,6 @@ internal fun notFoundIcError(what: String, libFile: KotlinLibraryFile? = null, s
icError("can not find $what", libFile, srcFile)
}
internal inline fun <E> buildListUntil(to: Int, builderAction: MutableList<E>.(Int) -> Unit): List<E> {
return buildList(to) { repeat(to) { builderAction(it) } }
}
internal inline fun <E> buildSetUntil(to: Int, builderAction: MutableSet<E>.(Int) -> Unit): Set<E> {
return HashSet<E>(to).apply { repeat(to) { builderAction(it) } }
}
@@ -6,17 +6,13 @@
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
import org.jetbrains.kotlin.ir.backend.js.jsOutputName
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.impl.javaFile
import org.jetbrains.kotlin.protobuf.CodedInputStream
import org.jetbrains.kotlin.protobuf.CodedOutputStream
import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
import java.io.File
internal class IncrementalCache(private val library: KotlinLibrary, val cacheDir: File) {
internal class IncrementalCache(private val library: KotlinLibraryHeader, val cacheDir: File) {
companion object {
private const val CACHE_HEADER = "ic.header.bin"
@@ -25,76 +21,71 @@ internal class IncrementalCache(private val library: KotlinLibrary, val cacheDir
private const val METADATA_TMP_SUFFIX = "metadata.tmp.bin"
}
private val signatureToIndexMappingFromMetadata = hashMapOf<KotlinSourceFile, MutableMap<IdSignature, Int>>()
private val libraryFile = KotlinLibraryFile(library)
private val cacheHeaderFile = File(cacheDir, CACHE_HEADER)
private var newCacheHeader: CacheHeader? = null
private var cacheHeaderShouldBeUpdated = false
private var removedSrcFiles: Collection<KotlinSourceFile> = emptyList()
private val kotlinLibraryHeader: KotlinLibraryHeader by lazy { KotlinLibraryHeader(library) }
private val kotlinLibrarySourceFileMetadata = hashMapOf<KotlinSourceFile, KotlinSourceFileMetadata>()
private val signatureToIndexMappingFromMetadata = hashMapOf<KotlinSourceFile, MutableMap<IdSignature, Int>>()
private val cacheHeaderFromDisk by lazy {
cacheHeaderFile.useCodedInputIfExists {
CacheHeader.fromProtoStream(this, library.libraryFingerprint)
}
}
val libraryFileFromHeader by lazy { cacheHeaderFromDisk?.libraryFile }
private class CacheHeader(
val libraryFile: KotlinLibraryFile,
private val libraryFingerprint: ICHash?,
val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>?
) {
constructor(library: KotlinLibraryHeader) : this(library.libraryFile, library.libraryFingerprint, library.sourceFileFingerprints)
private class CacheHeader(val klibFileHash: ICHash, val klibFile: KotlinLibraryFile, val fingerprints: Map<KotlinSourceFile, ICHash>?) {
fun toProtoStream(out: CodedOutputStream) {
klibFile.toProtoStream(out)
klibFileHash.toProtoStream(out)
libraryFile.toProtoStream(out)
if (fingerprints == null) {
notFoundIcError("fingerprints", klibFile)
}
out.writeInt32NoTag(fingerprints.size)
for ((srcFile, fingerprint) in fingerprints) {
srcFile.toProtoStream(out)
fingerprint.toProtoStream(out)
}
libraryFingerprint?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile)
sourceFileFingerprints?.let { fingerprints ->
out.writeInt32NoTag(fingerprints.size)
for ((srcFile, fingerprint) in fingerprints) {
srcFile.toProtoStream(out)
fingerprint.toProtoStream(out)
}
} ?: notFoundIcError("source file fingerprints", libraryFile)
}
companion object {
fun fromProtoStream(input: CodedInputStream, newKlibFileHash: ICHash): CacheHeader {
val klibFile = KotlinLibraryFile.fromProtoStream(input)
val oldKlibFileHash = ICHash.fromProtoStream(input)
fun fromProtoStream(input: CodedInputStream, newLibraryFingerprint: ICHash?): CacheHeader {
val libraryFile = KotlinLibraryFile.fromProtoStream(input)
val oldLibraryFingerprint = ICHash.fromProtoStream(input)
val fingerprints = (oldKlibFileHash != newKlibFileHash).ifTrue {
val sourceFileFingerprints = (oldLibraryFingerprint != newLibraryFingerprint).ifTrue {
buildMapUntil(input.readInt32()) {
val file = KotlinSourceFile.fromProtoStream(input)
put(file, ICHash.fromProtoStream(input))
}
}
return CacheHeader(oldKlibFileHash, klibFile, fingerprints)
return CacheHeader(libraryFile, oldLibraryFingerprint, sourceFileFingerprints)
}
}
}
private fun loadCacheHeader(newKlibFileHash: ICHash): CacheHeader? {
return cacheHeaderFile.useCodedInputIfExists {
val cacheHeader = CacheHeader.fromProtoStream(this, newKlibFileHash)
if (cacheHeader.klibFile != libraryFile) {
icError("broken incremental cache header $cacheHeaderFile; expected $libraryFile, got ${cacheHeader.klibFile}")
}
cacheHeader
}
}
private class KotlinSourceFileMetadataFromDisk(
override val inverseDependencies: KotlinSourceFileMap<Set<IdSignature>>,
override val directDependencies: KotlinSourceFileMap<Map<IdSignature, ICHash>>,
) : KotlinSourceFileMetadata()
private object KotlinSourceFileMetadataNotExist : KotlinSourceFileMetadata() {
override val inverseDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
override val directDependencies = KotlinSourceFileMap<Map<IdSignature, ICHash>>(emptyMap())
}
private val kotlinLibrarySourceFileMetadata = hashMapOf<KotlinSourceFile, KotlinSourceFileMetadata>()
private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix")
fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>): IncrementalCacheArtifact {
newCacheHeader?.let { cacheHeader ->
cacheHeaderFile.useCodedOutput { cacheHeader.toProtoStream(this) }
if (cacheHeaderShouldBeUpdated) {
cacheHeaderFile.useCodedOutput { CacheHeader(library).toProtoStream(this) }
}
for (removedFile in removedSrcFiles) {
@@ -102,51 +93,42 @@ internal class IncrementalCache(private val library: KotlinLibrary, val cacheDir
removedFile.getCacheFile(METADATA_SUFFIX).delete()
}
val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile ->
val fileArtifacts = library.sourceFileFingerprints.keys.map { srcFile ->
commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
}
return IncrementalCacheArtifact(cacheDir, removedSrcFiles.isNotEmpty(), fileArtifacts, library.jsOutputName)
}
data class ModifiedFiles(
val dirtyFiles: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
val addedFiles: List<KotlinSourceFile> = emptyList(),
val removedFiles: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
val newFiles: Set<KotlinSourceFile> = emptySet(),
val modifiedFiles: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
val nonModifiedFiles: List<KotlinSourceFile> = emptyList()
)
fun collectModifiedFiles(): ModifiedFiles {
val klibFileHash = library.libraryFile.javaFile().fileHashForIC()
val cachedFingerprints = cacheHeaderFromDisk?.let { it.sourceFileFingerprints ?: return ModifiedFiles() } ?: emptyMap()
val cacheHeader = loadCacheHeader(klibFileHash)
val addedFiles = mutableListOf<KotlinSourceFile>()
val modifiedFiles = hashMapOf<KotlinSourceFile, KotlinSourceFileMetadata>()
val nonModifiedFiles = mutableListOf<KotlinSourceFile>()
val cachedFingerprints = cacheHeader?.let { it.fingerprints ?: return ModifiedFiles() } ?: emptyMap()
val deletedFiles = HashSet(cachedFingerprints.keys)
val newFiles = mutableSetOf<KotlinSourceFile>()
val newFingerprints = HashMap<KotlinSourceFile, ICHash>(kotlinLibraryHeader.sourceFiles.size)
kotlinLibraryHeader.sourceFiles.forEachIndexed { index, file -> newFingerprints[file] = library.fingerprint(index) }
val modifiedFiles = HashMap<KotlinSourceFile, KotlinSourceFileMetadata>(newFingerprints.size).apply {
for ((file, fileNewFingerprint) in newFingerprints) {
val oldFingerprint = cachedFingerprints[file]
if (oldFingerprint == null) {
newFiles += file
}
if (oldFingerprint != fileNewFingerprint) {
val metadata = fetchSourceFileMetadata(file, false)
put(file, metadata)
}
deletedFiles.remove(file)
for ((file, fileNewFingerprint) in library.sourceFileFingerprints) {
when (cachedFingerprints[file]) {
fileNewFingerprint -> nonModifiedFiles.add(file)
null -> addedFiles.add(file)
else -> modifiedFiles[file] = fetchSourceFileMetadata(file, false)
}
}
val removedFiles = deletedFiles.associateWith { fetchSourceFileMetadata(it, false) }
val removedFiles = (cachedFingerprints.keys - library.sourceFileFingerprints.keys).associateWith {
fetchSourceFileMetadata(it, false)
}
removedSrcFiles = deletedFiles
newCacheHeader = CacheHeader(klibFileHash, libraryFile, newFingerprints)
removedSrcFiles = removedFiles.keys
cacheHeaderShouldBeUpdated = true
return ModifiedFiles(modifiedFiles, removedFiles, newFiles)
return ModifiedFiles(addedFiles, removedFiles, modifiedFiles, nonModifiedFiles)
}
fun fetchSourceFileFullMetadata(srcFile: KotlinSourceFile): KotlinSourceFileMetadata {
@@ -161,7 +143,7 @@ internal class IncrementalCache(private val library: KotlinLibrary, val cacheDir
kotlinLibrarySourceFileMetadata.getOrPut(srcFile) {
val signatureToIndexMapping = signatureToIndexMappingFromMetadata.getOrPut(srcFile) { hashMapOf() }
val deserializer: IdSignatureDeserializer by lazy {
kotlinLibraryHeader.signatureDeserializers[srcFile] ?: notFoundIcError("signature deserializer", libraryFile, srcFile)
library.sourceFileDeserializers[srcFile] ?: notFoundIcError("signature deserializer", library.libraryFile, srcFile)
}
fun CodedInputStream.deserializeIdSignatureAndSave() = readIdSignature { index ->
@@ -10,23 +10,45 @@ import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
import org.jetbrains.kotlin.ir.backend.js.jsOutputName
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import java.io.File
internal class KotlinLibraryHeader(library: KotlinLibrary) {
val sourceFiles: List<KotlinSourceFile>
val signatureDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer>
internal interface KotlinLibraryHeader {
val libraryFile: KotlinLibraryFile
init {
val libraryFingerprint: ICHash?
val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer>
val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>
val jsOutputName: String?
}
internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : KotlinLibraryHeader {
override val libraryFile: KotlinLibraryFile = KotlinLibraryFile(library)
override val libraryFingerprint: ICHash by lazy { File(libraryFile.path).fileHashForIC() }
override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> by lazy { sourceFiles.first }
override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> by lazy { sourceFiles.second }
override val jsOutputName: String?
get() = library.jsOutputName
private val sourceFiles by lazy {
val filesCount = library.fileCount()
val extReg = ExtensionRegistryLite.newInstance()
val files = buildListUntil(filesCount) {
val fileProto = IrFile.parseFrom(library.file(it).codedInputStream, extReg)
add(KotlinSourceFile(fileProto.fileEntry.name))
}
val deserializers = buildMapUntil(filesCount) {
put(files[it], IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
val deserializers = HashMap<KotlinSourceFile, IdSignatureDeserializer>(filesCount)
val fingerprints = HashMap<KotlinSourceFile, ICHash>(filesCount)
repeat(filesCount) {
val fileProto = IrFile.parseFrom(library.file(it).codedInputStream, extReg)
val srcFile = KotlinSourceFile(fileProto.fileEntry.name)
deserializers[srcFile] = IdSignatureDeserializer(IrLibraryFileFromBytes(object : IrLibraryBytesSource() {
private fun err(): Nothing = icError("Not supported")
override fun irDeclaration(index: Int): ByteArray = err()
override fun type(index: Int): ByteArray = err()
@@ -34,10 +56,22 @@ internal class KotlinLibraryHeader(library: KotlinLibrary) {
override fun string(index: Int): ByteArray = library.string(index, it)
override fun body(index: Int): ByteArray = err()
override fun debugInfo(index: Int): ByteArray? = null
}), null))
}
}), null)
sourceFiles = files
signatureDeserializers = deserializers
fingerprints[srcFile] = library.fingerprint(it)
}
deserializers to fingerprints
}
}
internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : KotlinLibraryHeader {
override val libraryFile: KotlinLibraryFile
get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}")
override val libraryFingerprint: ICHash? get() = null
override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> get() = emptyMap()
override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> get() = emptyMap()
override val jsOutputName: String? get() = null
}
@@ -98,9 +98,14 @@ abstract class KotlinSourceFileExports {
abstract class KotlinSourceFileMetadata : KotlinSourceFileExports() {
abstract val directDependencies: KotlinSourceFileMap<Map<IdSignature, ICHash>>
fun isEmpty() = inverseDependencies.isEmpty() && directDependencies.isEmpty()
}
fun KotlinSourceFileMetadata.isEmpty() = inverseDependencies.isEmpty() && directDependencies.isEmpty()
internal object KotlinSourceFileMetadataNotExist : KotlinSourceFileMetadata() {
override val inverseDependencies = KotlinSourceFileMap<Set<IdSignature>>(emptyMap())
override val directDependencies = KotlinSourceFileMap<Map<IdSignature, ICHash>>(emptyMap())
}
internal class DirtyFileExports(
override val inverseDependencies: KotlinSourceFileMutableMap<Set<IdSignature>> = KotlinSourceFileMutableMap()