[JS IR IC] Drop incremental cache after updating a klib set

Enabled Partial Linkage may replace entire IR expressions with a stub during klib linking and loading. This may break the incremental cache dependency graph. Dropping incremental cache files after updating klibs (e.g. update klib version) covers almost all cases which could not be tracked due to a broken dependency graph. This patch could be reverted after fixing KT-57347
2023-04-14 15:22:03 +02:00
parent c9badd14a7
commit 980d83eccd
22 changed files with 54 additions and 32 deletions
@@ -118,12 +118,28 @@ class CacheUpdater(
            libs.filter { it.libraryFile.canonicalPath in friendPaths }
        }

-        private val incrementalCaches = libraryDependencies.keys.associate { lib ->
+        private val klibCacheDirs = libraryDependencies.keys.map { lib ->
            val libFile = KotlinLibraryFile(lib)
            val file = File(libFile.path)
            val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX)
-            val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash")
-            libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
+            File(cacheRootDir, "${file.name}.$pathHash")
+        }
+
+        init {
+            // Enabled Partial Linkage may replace entire IR expressions with a stub during klib linking and loading.
+            // This may break the incremental cache dependency graph.
+            // Dropping incremental cache files after updating klibs (e.g. update klib version)
+            // covers almost all cases which could not be tracked due to a broken dependency graph.
+            // TODO: This code could be removed after fixing KT-57347
+            val oldKlibCaches = cacheRootDir.listFiles { file: File -> file.isDirectory }?.mapTo(hashSetOf()) { it.name } ?: emptySet()
+            val newKlibCaches = klibCacheDirs.mapTo(hashSetOf()) { it.name }
+            if (oldKlibCaches != newKlibCaches) {
+                cacheRootDir.deleteRecursively()
+            }
+        }
+
+        private val incrementalCaches = libraryDependencies.keys.zip(klibCacheDirs).associate { (lib, libraryCacheDir) ->
+            KotlinLibraryFile(lib) to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
        }

        private val removedIncrementalCaches = buildList {
@@ -161,15 +161,8 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend
            val moduleSourceDir = File(sourceDir, module)
            val moduleInfo = moduleInfos[module] ?: error("No module info found for $module")
            val moduleStep = moduleInfo.steps.getValue(projStepId)
-            val deletedFiles = mutableSetOf<String>()
            for (modification in moduleStep.modifications) {
-                modification.execute(moduleTestDir, moduleSourceDir) { deletedFiles.add(it.name) }
-            }
-
-            val expectedFileStats = moduleStep.expectedFileStats.toMutableMap()
-            if (deletedFiles.isNotEmpty()) {
-                val removedFiles = expectedFileStats[DirtyFileState.REMOVED_FILE.str] ?: emptySet()
-                expectedFileStats[DirtyFileState.REMOVED_FILE.str] = removedFiles + deletedFiles
+                modification.execute(moduleTestDir, moduleSourceDir) {}
            }

            val outputKlibFile = resolveModuleArtifact(module, buildDir)
@@ -196,7 +189,7 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend
                module.safeModuleName,
                outputKlibFile.canonicalPath,
                friends.map { it.canonicalPath },
-                expectedFileStats,
+                moduleStep.expectedFileStats,
                dtsFile?.readText()
            )
        }
@@ -25,5 +25,6 @@ STEP 6:
    modifications:
        D : l1b.kt
    removed inverse depends: l1a.kt
+    removed file: l1b.kt
 STEP 7:
    updated exports: l1a.kt
@@ -1,11 +1,11 @@
 STEP 0:
    added file: l1a.kt, l1b.kt
 STEP 1:
-    updated exports: l1b.kt, l1a.kt
+    added file: l1b.kt, l1a.kt
 STEP 2:
    updated exports: l1a.kt
 STEP 3:
-    removed inverse depends: l1b.kt, l1a.kt
+    added file: l1b.kt, l1a.kt
 STEP 4:
    modifications:
        U : l1a.4.kt -> l1a.kt
@@ -7,7 +7,7 @@ STEP 1:
    dependencies: lib1, lib2
    modifications:
        U : m.1.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 2:
    dependencies: lib1, lib2
    updated imports: m.kt
@@ -15,6 +15,6 @@ STEP 3:
    dependencies: lib1
    modifications:
        U : m.0.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 4:
    dependencies: lib1
@@ -42,6 +42,7 @@ STEP 7:
        D : l2pair.kt
        D : l2string.kt
        D : l2func.kt
+    removed file: l2pair.kt, l2string.kt, l2func.kt
 STEP 8:
    dependencies: lib1
    updated imports: l2int.kt
@@ -33,7 +33,7 @@ STEP 7:
 STEP 8:
    modifications:
        U : l1.0.kt -> l1.kt
-    modified ir: l1.kt
+    added file: l1.kt
 STEP 9:
    modifications:
        U : l1.1.kt -> l1.kt
@@ -14,7 +14,7 @@ STEP 8:
    dependencies: lib1, libProxy
    modifications:
        U : m.proxy.kt -> m.kt
-    modified ir: m.kt
+    added file: m.kt
 STEP 9..13:
    dependencies: lib1, libProxy
    updated imports: m.kt
@@ -3,6 +3,7 @@ STEP 0:
 STEP 1:
    modifications:
        D : l12.kt
+    removed file: l12.kt
 STEP 2:
    modifications:
        U : l12_n.2.kt -> l12_n.kt
@@ -12,3 +12,4 @@ STEP 2:
    modifications:
        D : l22.kt
    removed direct depends: l21.kt
+    removed file: l22.kt
@@ -3,3 +3,4 @@ STEP 0:
 STEP 1:
    modifications:
        D : l1_1.kt
+    removed file: l1_1.kt
@@ -5,3 +5,4 @@ STEP 1:
    dependencies: lib1
    modifications:
        D : l2_1.kt
+    removed file: l2_1.kt
@@ -3,4 +3,5 @@ STEP 0:
 STEP 1:
    modifications:
        D : l1_1.kt
+    removed file: l1_1.kt
 STEP 2:
@@ -5,3 +5,4 @@ STEP 1:
        D : l1.kt
        U : l1.kt -> l1_new.kt
    added file: l1_new.kt
+    removed file: l1.kt
@@ -7,6 +7,4 @@ STEP 2:
    modifications:
        U : l1.2.kt -> l1.kt
    modified ir: l1.kt
-STEP 3:
-    removed file: l1.kt
-STEP 4..5:
+STEP 3..5:
@@ -2,9 +2,7 @@ STEP 0:
    modifications:
        U : l1.0.kt -> l1.kt
    added file: l1.kt
-STEP 1:
-    removed file: l1.kt
-STEP 2:
+STEP 1..2:
 STEP 3:
    modifications:
        U : l1.3.kt -> l1.kt
@@ -3,13 +3,13 @@ STEP 0:
    added file: m.kt
 STEP 1:
    dependencies: lib1-new
-    removed direct depends: m.kt
+    added file: m.kt
 STEP 2:
    dependencies: lib1-new
    updated imports: m.kt
 STEP 3:
    dependencies: lib1
-    removed direct depends: m.kt
+    added file: m.kt
 STEP 4..5:
    dependencies: lib1
    updated imports: m.kt
@@ -5,12 +5,12 @@ STEP 0:
    added file: l2.kt
 STEP 1:
    dependencies: lib1-a, lib1-b, lib1-c
-    removed direct depends: l2.kt
+    added file: l2.kt
 STEP 2..3:
    dependencies: lib1-a, lib1-b, lib1-c
    updated imports: l2.kt
 STEP 4:
    dependencies: lib1
-    removed direct depends: l2.kt
+    added file: l2.kt
 STEP 5:
    dependencies: lib1
@@ -1,7 +1,13 @@
 STEP 0:
    dependencies: lib1, lib2
    added file: m.kt
-STEP 1..3:
+STEP 1:
    dependencies: lib1-a, lib1-b, lib1-c, lib2
-STEP 4..5:
+    added file: m.kt
+STEP 2..3:
+    dependencies: lib1-a, lib1-b, lib1-c, lib2
+STEP 4:
+    dependencies: lib1, lib2
+    added file: m.kt
+STEP 5:
    dependencies: lib1, lib2
@@ -5,13 +5,13 @@ STEP 0:
    dirty js: lib1, lib2, main
 STEP 1:
    libs: lib1-a, lib1-b, lib1-c, lib2, main
-    dirty js: lib1-a, lib1-b, lib1-c, lib2
+    dirty js: lib1-a, lib1-b, lib1-c, main, lib2
 STEP 2..3:
    libs: lib1-a, lib1-b, lib1-c, lib2, main
    dirty js: lib1-b, lib2
 STEP 4:
    libs: lib1, lib2, main
-    dirty js: lib1, lib2
+    dirty js: lib1, lib2, main
 STEP 5:
    libs: lib1, lib2, main
    dirty js: lib1
@@ -32,6 +32,7 @@ STEP 5:
        U : l2b.5.kt -> l2b.kt
        D : l2a.kt
    modified ir: l2b.kt
+    removed file: l2a.kt
 STEP 6:
    dependencies: lib1
    modifications:
@@ -43,6 +44,7 @@ STEP 7:
        U : l2a.7.kt -> l2a.kt
        D : l2b.kt
    added file: l2a.kt
+    removed file: l2b.kt
 STEP 8:
    dependencies: lib1
    modifications:
@@ -26,6 +26,7 @@ STEP 5:
        U : l2b.5.kt -> l2b.kt
        D : l2a.kt
    modified ir: l2b.kt
+    removed file: l2a.kt
 STEP 6:
    dependencies: lib1
    updated imports: l2b.kt