[JS IR] Commit cache header after lowering

Commiting cache header before lowering may
 break caches in case of lowering errors.

^KT-54912 Fixed

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -650,7 +650,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             val cacheUpdater = CacheUpdater(
                 mainModule = arguments.includes!!,
                 allModules = libraries,
-                icCacheRootDir = cacheDirectories.last(),
+                cacheDir = cacheDirectories.last(),
                 compilerConfiguration = configurationJs,
                 irFactory = { IrFactoryImplForJsIC(WholeWorldStageController()) },
                 mainArguments = mainCallArguments,
@@ -666,8 +666,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
             var libIndex = 0
             for ((libFile, srcFiles) in cacheUpdater.getDirtyFileLastStats()) {
                 val (msg, showFiles) = when {
-                    srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) } -> "fully rebuilt due to clean build" to false
+                    srcFiles.values.all { it.contains(DirtyFileState.ADDED_FILE) } -> "fully rebuilt" to false
-                    srcFiles.values.all { it.contains(DirtyFileState.MODIFIED_CONFIG) } -> "fully rebuilt due to config modification" to false
                     else -> "partially rebuilt" to true
                 }
                 messageCollector.report(INFO, "${++libIndex}) module [${File(libFile.path).name}] was $msg")

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -37,7 +37,6 @@ fun interface JsIrCompilerICInterfaceFactory {
 
 enum class DirtyFileState(val str: String) {
     ADDED_FILE("added file"),
-    MODIFIED_CONFIG("modified config"),
     MODIFIED_IR("modified ir"),
     UPDATED_EXPORTS("updated exports"),
     UPDATED_IMPORTS("updated imports"),
@@ -49,7 +48,7 @@ enum class DirtyFileState(val str: String) {
 class CacheUpdater(
     mainModule: String,
     private val allModules: Collection<String>,
-    private val icCacheRootDir: String,
+    cacheDir: String,
     private val compilerConfiguration: CompilerConfiguration,
     private val irFactory: () -> IrFactory,
     private val mainArguments: List<String>?,
@@ -59,10 +58,10 @@ class CacheUpdater(
 
     private val dirtyFileStats = KotlinSourceFileMutableMap<EnumSet<DirtyFileState>>()
 
-    private val configHash = compilerConfiguration.configHashForIC()
-
     private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)
 
+    private val cacheRootDir = File(cacheDir, "version.${compilerConfiguration.configHashForIC()}")
+
     fun getDirtyFileLastStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats
 
     fun getStopwatchLastLaps() = stopwatch.laps
@@ -104,12 +103,10 @@ class CacheUpdater(
             }
         }
 
-        private val incrementalCaches = stopwatch.measure("Incremental cache - initializing") {
+        private val incrementalCaches = libraryDependencies.keys.associate { lib ->
-            libraryDependencies.keys.associate { lib ->
+            val libFile = KotlinLibraryFile(lib)
-                val libFile = KotlinLibraryFile(lib)
+            val file = File(libFile.path)
-                val file = File(libFile.path)
+            libFile to IncrementalCache(lib, File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}"))
-                libFile to IncrementalCache(lib, File(icCacheRootDir, "${file.absolutePath.stringHashForIC()}.${file.name}"))
-            }
         }
 
         private fun getLibIncrementalCache(libFile: KotlinLibraryFile) =
@@ -159,14 +156,13 @@ class CacheUpdater(
             val removedFilesMetadata = hashMapOf<KotlinLibraryFile, Map<KotlinSourceFile, KotlinSourceFileMetadata>>()
 
             val modifiedFiles = KotlinSourceFileMap(incrementalCaches.entries.associate { (lib, cache) ->
-                val (dirtyFiles, removedFiles, newFiles, modifiedConfigFiles) = cache.collectModifiedFiles(configHash)
+                val (dirtyFiles, removedFiles, newFiles) = cache.collectModifiedFiles()
 
                 val fileStats by lazy(LazyThreadSafetyMode.NONE) { dirtyFileStats.getOrPutFiles(lib) }
                 newFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.ADDED_FILE) }
-                modifiedConfigFiles.forEach { fileStats.addDirtFileStat(it, DirtyFileState.MODIFIED_CONFIG) }
                 removedFiles.forEach { fileStats.addDirtFileStat(it.key, DirtyFileState.REMOVED_FILE) }
                 dirtyFiles.forEach {
-                    if (it.key !in newFiles && it.key !in modifiedConfigFiles) {
+                    if (it.key !in newFiles) {
                         fileStats.addDirtFileStat(it.key, DirtyFileState.MODIFIED_IR)
                     }
                 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -12,10 +12,11 @@ import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.impl.javaFile
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
+import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
 import java.io.File
 
 
-internal class IncrementalCache(private val library: KotlinLibrary, private val cacheDir: File) {
+internal class IncrementalCache(private val library: KotlinLibrary, val cacheDir: File) {
     companion object {
         private const val CACHE_HEADER = "ic.header.bin"
 
@@ -24,40 +25,58 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val
         private const val METADATA_TMP_SUFFIX = "metadata.tmp.bin"
     }
 
-    private var forceRebuildJs = false
     private val signatureToIndexMappingFromMetadata = hashMapOf<KotlinSourceFile, MutableMap<IdSignature, Int>>()
 
     private val libraryFile = KotlinLibraryFile(library)
 
-    class CacheHeader(val klibFileHash: ICHash = ICHash(), val configHash: ICHash = ICHash()) {
+    private val cacheHeaderFile = File(cacheDir, CACHE_HEADER)
+
+    private var newCacheHeader: CacheHeader? = null
+
+    private var removedSrcFiles: Collection<KotlinSourceFile> = emptyList()
+
+    private val kotlinLibraryHeader: KotlinLibraryHeader by lazy { KotlinLibraryHeader(library) }
+
+    private class CacheHeader(val klibFileHash: ICHash, val klibFile: KotlinLibraryFile, val fingerprints: Map<KotlinSourceFile, ICHash>?) {
         fun toProtoStream(out: CodedOutputStream) {
+            klibFile.toProtoStream(out)
             klibFileHash.toProtoStream(out)
-            configHash.toProtoStream(out)
+
+            if (fingerprints == null) {
+                notFoundIcError("fingerprints", klibFile)
+            }
+            out.writeInt32NoTag(fingerprints.size)
+            for ((srcFile, fingerprint) in fingerprints) {
+                srcFile.toProtoStream(out)
+                fingerprint.toProtoStream(out)
+            }
         }
 
         companion object {
-            fun fromProtoStream(input: CodedInputStream): CacheHeader {
+            fun fromProtoStream(input: CodedInputStream, newKlibFileHash: ICHash): CacheHeader {
-                val klibFileHash = ICHash.fromProtoStream(input)
+                val klibFile = KotlinLibraryFile.fromProtoStream(input)
-                val configHash = ICHash.fromProtoStream(input)
+                val oldKlibFileHash = ICHash.fromProtoStream(input)
-                return CacheHeader(klibFileHash, configHash)
+
+                val fingerprints = (oldKlibFileHash != newKlibFileHash).ifTrue {
+                    buildMapUntil(input.readInt32()) {
+                        val file = KotlinSourceFile.fromProtoStream(input)
+                        put(file, ICHash.fromProtoStream(input))
+                    }
+                }
+                return CacheHeader(oldKlibFileHash, klibFile, fingerprints)
             }
         }
     }
 
-    private var cacheHeader = File(cacheDir, CACHE_HEADER).useCodedInputIfExists {
+    private fun loadCacheHeader(newKlibFileHash: ICHash): CacheHeader? {
-        CacheHeader.fromProtoStream(this)
+        return cacheHeaderFile.useCodedInputIfExists {
-    } ?: CacheHeader()
+            val cacheHeader = CacheHeader.fromProtoStream(this, newKlibFileHash)
-
+            if (cacheHeader.klibFile != libraryFile) {
-    private fun loadCachedFingerprints() = File(cacheDir, CACHE_HEADER).useCodedInputIfExists {
+                icError("broken incremental cache header $cacheHeaderFile; expected $libraryFile, got ${cacheHeader.klibFile}")
-        // skip cache header
+            }
-        CacheHeader.fromProtoStream(this@useCodedInputIfExists)
+            cacheHeader
-        buildMapUntil(readInt32()) {
-            val file = KotlinSourceFile.fromProtoStream(this@useCodedInputIfExists)
-            put(file, ICHash.fromProtoStream(this@useCodedInputIfExists))
         }
-    } ?: emptyMap()
+    }
-
-    private val kotlinLibraryHeader: KotlinLibraryHeader by lazy { KotlinLibraryHeader(library) }
 
     private class KotlinSourceFileMetadataFromDisk(
         override val inverseDependencies: KotlinSourceFileMap<Set<IdSignature>>,
@@ -73,53 +92,46 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val
 
     private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix")
 
-    private fun commitCacheHeader(fingerprints: List<Pair<KotlinSourceFile, ICHash>>) = File(cacheDir, CACHE_HEADER).useCodedOutput {
-        cacheHeader.toProtoStream(this)
-        writeInt32NoTag(fingerprints.size)
-        for ((srcFile, fingerprint) in fingerprints) {
-            srcFile.toProtoStream(this)
-            fingerprint.toProtoStream(this)
-        }
-    }
-
     fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>): IncrementalCacheArtifact {
+        newCacheHeader?.let { cacheHeader ->
+            cacheHeaderFile.useCodedOutput { cacheHeader.toProtoStream(this) }
+        }
+
+        for (removedFile in removedSrcFiles) {
+            removedFile.getCacheFile(BINARY_AST_SUFFIX).delete()
+            removedFile.getCacheFile(METADATA_SUFFIX).delete()
+        }
+
         val fileArtifacts = kotlinLibraryHeader.sourceFiles.map { srcFile ->
             commitSourceFileMetadata(srcFile.getCacheFile(BINARY_AST_SUFFIX), srcFile, signatureToIndexMapping[srcFile] ?: emptyMap())
         }
-        return IncrementalCacheArtifact(cacheDir, forceRebuildJs, fileArtifacts, library.jsOutputName)
+        return IncrementalCacheArtifact(cacheDir, removedSrcFiles.isNotEmpty(), fileArtifacts, library.jsOutputName)
     }
 
     data class ModifiedFiles(
         val dirtyFiles: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
         val removedFiles: Map<KotlinSourceFile, KotlinSourceFileMetadata> = emptyMap(),
         val newFiles: Set<KotlinSourceFile> = emptySet(),
-        val modifiedConfigFiles: Set<KotlinSourceFile> = emptySet(),
     )
 
-    fun collectModifiedFiles(configHash: ICHash): ModifiedFiles {
+    fun collectModifiedFiles(): ModifiedFiles {
-        var isConfigModified = false
         val klibFileHash = library.libraryFile.javaFile().fileHashForIC()
-        cacheHeader = when {
-            cacheHeader.configHash != configHash -> {
-                cacheDir.deleteRecursively()
-                isConfigModified = cacheHeader.configHash != ICHash()
-                CacheHeader(klibFileHash, configHash)
-            }
 
-            cacheHeader.klibFileHash != klibFileHash -> CacheHeader(klibFileHash, configHash)
+        val cacheHeader = loadCacheHeader(klibFileHash)
-            else -> return ModifiedFiles()
+
-        }
+        val cachedFingerprints = cacheHeader?.let { it.fingerprints ?: return ModifiedFiles() } ?: emptyMap()
 
-        val cachedFingerprints = loadCachedFingerprints()
         val deletedFiles = HashSet(cachedFingerprints.keys)
-        val unknownFiles = mutableSetOf<KotlinSourceFile>()
+        val newFiles = mutableSetOf<KotlinSourceFile>()
+
+        val newFingerprints = HashMap<KotlinSourceFile, ICHash>(kotlinLibraryHeader.sourceFiles.size)
+        kotlinLibraryHeader.sourceFiles.forEachIndexed { index, file -> newFingerprints[file] = library.fingerprint(index) }
 
-        val newFingerprints = kotlinLibraryHeader.sourceFiles.mapIndexed { index, file -> file to library.fingerprint(index) }
         val modifiedFiles = HashMap<KotlinSourceFile, KotlinSourceFileMetadata>(newFingerprints.size).apply {
             for ((file, fileNewFingerprint) in newFingerprints) {
                 val oldFingerprint = cachedFingerprints[file]
                 if (oldFingerprint == null) {
-                    unknownFiles += file
+                    newFiles += file
                 }
                 if (oldFingerprint != fileNewFingerprint) {
                     val metadata = fetchSourceFileMetadata(file, false)
@@ -129,23 +141,12 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val
             }
         }
 
-        val removedFilesMetadata = deletedFiles.associateWith {
+        val removedFiles = deletedFiles.associateWith { fetchSourceFileMetadata(it, false) }
-            val metadata = fetchSourceFileMetadata(it, false)
-            it.getCacheFile(BINARY_AST_SUFFIX).delete()
-            it.getCacheFile(METADATA_SUFFIX).delete()
-            metadata
-        }
 
-        forceRebuildJs = deletedFiles.isNotEmpty()
+        removedSrcFiles = deletedFiles
-        commitCacheHeader(newFingerprints)
+        newCacheHeader = CacheHeader(klibFileHash, libraryFile, newFingerprints)
 
-        val (newFiles, modifiedConfigFiles) = if (isConfigModified) {
+        return ModifiedFiles(modifiedFiles, removedFiles, newFiles)
-            emptySet<KotlinSourceFile>() to unknownFiles
-        } else {
-            unknownFiles to emptySet<KotlinSourceFile>()
-        }
-
-        return ModifiedFiles(modifiedFiles, removedFilesMetadata, newFiles, modifiedConfigFiles)
     }
 
     fun fetchSourceFileFullMetadata(srcFile: KotlinSourceFile): KotlinSourceFileMetadata {
@@ -170,12 +171,12 @@ internal class IncrementalCache(private val library: KotlinLibrary, private val
             }
 
             fun <T> CodedInputStream.readDependencies(signaturesReader: () -> T) = buildMapUntil(readInt32()) {
-                val libraryFile = KotlinLibraryFile.fromProtoStream(this@readDependencies)
+                val libFile = KotlinLibraryFile.fromProtoStream(this@readDependencies)
                 val depends = buildMapUntil(readInt32()) {
                     val dependencySrcFile = KotlinSourceFile.fromProtoStream(this@readDependencies)
                     put(dependencySrcFile, signaturesReader())
                 }
-                put(libraryFile, depends)
+                put(libFile, depends)
             }
 
             fun CodedInputStream.readDirectDependencies() = readDependencies {

js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt

@@ -229,7 +229,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
                 val cacheUpdater = CacheUpdater(
                     mainModule = testInfo.last().modulePath,
                     allModules = testInfo.mapTo(mutableListOf(STDLIB_KLIB)) { it.modulePath },
-                    icCacheRootDir = buildDir.resolve("incremental-cache").absolutePath,
+                    cacheDir = buildDir.resolve("incremental-cache").absolutePath,
                     compilerConfiguration = configuration,
                     irFactory = { IrFactoryImplForJsIC(WholeWorldStageController()) },
                     mainArguments = null,

js/js.tests/test/org/jetbrains/kotlin/js/test/ir/AbstractJsKLibABITestCase.kt

@@ -193,7 +193,7 @@ abstract class AbstractJsKLibABITestCase : KtUsefulTestCase() {
         val cacheUpdater = CacheUpdater(
             mainModule = mainModuleKlibFile.absolutePath,
             allModules = allDependencies.regularDependencies.map { it.path },
-            icCacheRootDir = buildDir.resolve("libs-cache").absolutePath,
+            cacheDir = buildDir.resolve("libs-cache").absolutePath,
             compilerConfiguration = configuration,
             irFactory = { IrFactoryImplForJsIC(WholeWorldStageController()) },
             mainArguments = null,

js/js.translator/testData/incremental/invalidation/enum/lib1/module.info

@@ -7,6 +7,6 @@ STEP 1:
         U : l1.1.kt -> l1.kt
     modified ir: l1.kt
 STEP 2:
-    modified config: l1.kt
+    added file: l1.kt
 STEP 3:
     updated exports: l1.kt

js/js.translator/testData/incremental/invalidation/enum/main/module.info

@@ -7,7 +7,7 @@ STEP 1:
     dependencies: lib1
 STEP 2:
     dependencies: lib1
-    modified config: m.kt, testEnumEntries.kt, testEnumValues.kt
+    added file: m.kt, testEnumEntries.kt, testEnumValues.kt
 STEP 3:
     modifications:
         U : testEnumEntries.3.kt -> testEnumEntries.kt

libraries/tools/kotlin-gradle-plugin-integration-tests/src/test/kotlin/org/jetbrains/kotlin/gradle/Kotlin2JsGradlePluginIT.kt

@@ -176,18 +176,23 @@ class Kotlin2JsIrGradlePluginIT : AbstractKotlin2JsGradlePluginIT(true) {
     fun testJsIrIncrementalMultipleArtifacts(gradleVersion: GradleVersion) {
         project("kotlin-js-ir-ic-multiple-artifacts", gradleVersion) {
             build("compileDevelopmentExecutableKotlinJs") {
-                val cacheDir = projectPath.resolve("app/build/klib/cache/lib/")
+                val cacheDir = projectPath.resolve("app/build/klib/cache/").toFile()
-                    .toFile()
+                val cacheRootDirName = cacheDir.list()?.singleOrNull()
-                assertTrue("Lib cache size should be 2") {
+                assertTrue("Lib cache root dir should contain 1 element 'version.hash'") {
-                    cacheDir
+                    cacheRootDirName?.startsWith("version.") ?: false
-                        .list()
-                        ?.size == 2
                 }
+                val cacheRootDir = cacheDir.resolve(cacheRootDirName!!)
+                val klibCacheDirs = cacheRootDir.list()
+                // 2 for lib.klib + 1 for stdlib + 1 for main
+                assertEquals(4, klibCacheDirs?.size, "cache should contain 4 dirs")
+
+                val libKlibCacheDirs = klibCacheDirs?.filter { dir -> dir.startsWith("lib.klib.") }
+                assertEquals(2, libKlibCacheDirs?.size, "cache should contain 2 dirs for lib.klib")
 
                 var lib = false
                 var libOther = false
 
-                cacheDir.listFiles()!!
+                cacheRootDir.listFiles()!!
                     .forEach {
                         it.listFiles()!!
                             .filter { it.isFile }