[JS IR IC] Drop incremental cache after updating a klib set

Enabled Partial Linkage may replace entire IR expressions with a stub
during klib linking and loading. This may break the incremental cache
dependency graph. Dropping incremental cache files after updating klibs
(e.g. update klib version) covers almost all cases which
could not be tracked due to a broken dependency graph.

This patch could be reverted after fixing KT-57347

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -118,12 +118,28 @@ class CacheUpdater(
             libs.filter { it.libraryFile.canonicalPath in friendPaths }
         }
 
-        private val incrementalCaches = libraryDependencies.keys.associate { lib ->
+        private val klibCacheDirs = libraryDependencies.keys.map { lib ->
             val libFile = KotlinLibraryFile(lib)
             val file = File(libFile.path)
             val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX)
-            val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash")
+            File(cacheRootDir, "${file.name}.$pathHash")
-            libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
+        }
+
+        init {
+            // Enabled Partial Linkage may replace entire IR expressions with a stub during klib linking and loading.
+            // This may break the incremental cache dependency graph.
+            // Dropping incremental cache files after updating klibs (e.g. update klib version)
+            // covers almost all cases which could not be tracked due to a broken dependency graph.
+            // TODO: This code could be removed after fixing KT-57347
+            val oldKlibCaches = cacheRootDir.listFiles { file: File -> file.isDirectory }?.mapTo(hashSetOf()) { it.name } ?: emptySet()
+            val newKlibCaches = klibCacheDirs.mapTo(hashSetOf()) { it.name }
+            if (oldKlibCaches != newKlibCaches) {
+                cacheRootDir.deleteRecursively()
+            }
+        }
+
+        private val incrementalCaches = libraryDependencies.keys.zip(klibCacheDirs).associate { (lib, libraryCacheDir) ->
+            KotlinLibraryFile(lib) to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
         }
 
         private val removedIncrementalCaches = buildList {

js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt

@@ -161,15 +161,8 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend
             val moduleSourceDir = File(sourceDir, module)
             val moduleInfo = moduleInfos[module] ?: error("No module info found for $module")
             val moduleStep = moduleInfo.steps.getValue(projStepId)
-            val deletedFiles = mutableSetOf<String>()
             for (modification in moduleStep.modifications) {
-                modification.execute(moduleTestDir, moduleSourceDir) { deletedFiles.add(it.name) }
+                modification.execute(moduleTestDir, moduleSourceDir) {}
-            }
-
-            val expectedFileStats = moduleStep.expectedFileStats.toMutableMap()
-            if (deletedFiles.isNotEmpty()) {
-                val removedFiles = expectedFileStats[DirtyFileState.REMOVED_FILE.str] ?: emptySet()
-                expectedFileStats[DirtyFileState.REMOVED_FILE.str] = removedFiles + deletedFiles
             }
 
             val outputKlibFile = resolveModuleArtifact(module, buildDir)
@@ -196,7 +189,7 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend
                 module.safeModuleName,
                 outputKlibFile.canonicalPath,
                 friends.map { it.canonicalPath },
-                expectedFileStats,
+                moduleStep.expectedFileStats,
                 dtsFile?.readText()
             )
         }

js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentFile/lib1/module.info

@@ -25,5 +25,6 @@ STEP 6:
     modifications:
         D : l1b.kt
     removed inverse depends: l1a.kt
+    removed file: l1b.kt
 STEP 7:
     updated exports: l1a.kt

js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/lib1/module.info

@@ -1,11 +1,11 @@
 STEP 0:
     added file: l1a.kt, l1b.kt
 STEP 1:
-    updated exports: l1b.kt, l1a.kt
+    added file: l1b.kt, l1a.kt
 STEP 2:
     updated exports: l1a.kt
 STEP 3:
-    removed inverse depends: l1b.kt, l1a.kt
+    added file: l1b.kt, l1a.kt
 STEP 4:
     modifications:
         U : l1a.4.kt -> l1a.kt

js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/main/module.info

@@ -7,7 +7,7 @@ STEP 1:
     dependencies: lib1, lib2
     modifications:
         U : m.1.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 2:
     dependencies: lib1, lib2
     updated imports: m.kt
@@ -15,6 +15,6 @@ STEP 3:
     dependencies: lib1
     modifications:
         U : m.0.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 4:
     dependencies: lib1

js/js.translator/testData/incremental/invalidation/companionProperties/lib2/module.info

@@ -42,6 +42,7 @@ STEP 7:
         D : l2pair.kt
         D : l2string.kt
         D : l2func.kt
+    removed file: l2pair.kt, l2string.kt, l2func.kt
 STEP 8:
     dependencies: lib1
     updated imports: l2int.kt

js/js.translator/testData/incremental/invalidation/genericInlineFunctions/lib1/module.info

@@ -33,7 +33,7 @@ STEP 7:
 STEP 8:
     modifications:
         U : l1.0.kt -> l1.kt
-    modified ir: l1.kt
+    added file: l1.kt
 STEP 9:
     modifications:
         U : l1.1.kt -> l1.kt

js/js.translator/testData/incremental/invalidation/genericInlineFunctions/main/module.info

@@ -14,7 +14,7 @@ STEP 8:
     dependencies: lib1, libProxy
     modifications:
         U : m.proxy.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 9..13:
     dependencies: lib1, libProxy
     updated imports: m.kt

js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib1/module.info

@@ -3,6 +3,7 @@ STEP 0:
 STEP 1:
     modifications:
         D : l12.kt
+    removed file: l12.kt
 STEP 2:
     modifications:
         U : l12_n.2.kt -> l12_n.kt

js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib2/module.info

@@ -12,3 +12,4 @@ STEP 2:
     modifications:
         D : l22.kt
     removed direct depends: l21.kt
+    removed file: l22.kt

js/js.translator/testData/incremental/invalidation/removeFile/lib1/module.info

@@ -3,3 +3,4 @@ STEP 0:
 STEP 1:
     modifications:
         D : l1_1.kt
+    removed file: l1_1.kt

js/js.translator/testData/incremental/invalidation/removeFile/lib2/module.info

@@ -5,3 +5,4 @@ STEP 1:
     dependencies: lib1
     modifications:
         D : l2_1.kt
+    removed file: l2_1.kt

js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info

@@ -3,4 +3,5 @@ STEP 0:
 STEP 1:
     modifications:
         D : l1_1.kt
+    removed file: l1_1.kt
 STEP 2:

js/js.translator/testData/incremental/invalidation/renameFile/lib1/module.info

@@ -5,3 +5,4 @@ STEP 1:
         D : l1.kt
         U : l1.kt -> l1_new.kt
     added file: l1_new.kt
+    removed file: l1.kt

js/js.translator/testData/incremental/invalidation/renameModule/lib1-new/module.info

@@ -7,6 +7,4 @@ STEP 2:
     modifications:
         U : l1.2.kt -> l1.kt
     modified ir: l1.kt
-STEP 3:
+STEP 3..5:
-    removed file: l1.kt
-STEP 4..5:

js/js.translator/testData/incremental/invalidation/renameModule/lib1/module.info

@@ -2,9 +2,7 @@ STEP 0:
     modifications:
         U : l1.0.kt -> l1.kt
     added file: l1.kt
-STEP 1:
+STEP 1..2:
-    removed file: l1.kt
-STEP 2:
 STEP 3:
     modifications:
         U : l1.3.kt -> l1.kt

js/js.translator/testData/incremental/invalidation/renameModule/main/module.info

@@ -3,13 +3,13 @@ STEP 0:
     added file: m.kt
 STEP 1:
     dependencies: lib1-new
-    removed direct depends: m.kt
+    added file: m.kt
 STEP 2:
     dependencies: lib1-new
     updated imports: m.kt
 STEP 3:
     dependencies: lib1
-    removed direct depends: m.kt
+    added file: m.kt
 STEP 4..5:
     dependencies: lib1
     updated imports: m.kt

js/js.translator/testData/incremental/invalidation/splitJoinModule/lib2/module.info

@@ -5,12 +5,12 @@ STEP 0:
     added file: l2.kt
 STEP 1:
     dependencies: lib1-a, lib1-b, lib1-c
-    removed direct depends: l2.kt
+    added file: l2.kt
 STEP 2..3:
     dependencies: lib1-a, lib1-b, lib1-c
     updated imports: l2.kt
 STEP 4:
     dependencies: lib1
-    removed direct depends: l2.kt
+    added file: l2.kt
 STEP 5:
     dependencies: lib1

js/js.translator/testData/incremental/invalidation/splitJoinModule/main/module.info

@@ -1,7 +1,13 @@
 STEP 0:
     dependencies: lib1, lib2
     added file: m.kt
-STEP 1..3:
+STEP 1:
     dependencies: lib1-a, lib1-b, lib1-c, lib2
-STEP 4..5:
+    added file: m.kt
+STEP 2..3:
+    dependencies: lib1-a, lib1-b, lib1-c, lib2
+STEP 4:
+    dependencies: lib1, lib2
+    added file: m.kt
+STEP 5:
     dependencies: lib1, lib2

js/js.translator/testData/incremental/invalidation/splitJoinModule/project.info

@@ -5,13 +5,13 @@ STEP 0:
     dirty js: lib1, lib2, main
 STEP 1:
     libs: lib1-a, lib1-b, lib1-c, lib2, main
-    dirty js: lib1-a, lib1-b, lib1-c, lib2
+    dirty js: lib1-a, lib1-b, lib1-c, main, lib2
 STEP 2..3:
     libs: lib1-a, lib1-b, lib1-c, lib2, main
     dirty js: lib1-b, lib2
 STEP 4:
     libs: lib1, lib2, main
-    dirty js: lib1, lib2
+    dirty js: lib1, lib2, main
 STEP 5:
     libs: lib1, lib2, main
     dirty js: lib1

js/js.translator/testData/incremental/invalidation/updateExports/lib2/module.info

@@ -32,6 +32,7 @@ STEP 5:
         U : l2b.5.kt -> l2b.kt
         D : l2a.kt
     modified ir: l2b.kt
+    removed file: l2a.kt
 STEP 6:
     dependencies: lib1
     modifications:
@@ -43,6 +44,7 @@ STEP 7:
         U : l2a.7.kt -> l2a.kt
         D : l2b.kt
     added file: l2a.kt
+    removed file: l2b.kt
 STEP 8:
     dependencies: lib1
     modifications:

js/js.translator/testData/incremental/invalidation/updateExportsAndInlineImports/lib2/module.info

@@ -26,6 +26,7 @@ STEP 5:
         U : l2b.5.kt -> l2b.kt
         D : l2a.kt
     modified ir: l2b.kt
+    removed file: l2a.kt
 STEP 6:
     dependencies: lib1
     updated imports: l2b.kt