From 980d83eccda18229d8f6fd3aa95008258265e489 Mon Sep 17 00:00:00 2001 From: Alexander Korepanov Date: Fri, 14 Apr 2023 15:22:03 +0200 Subject: [PATCH] [JS IR IC] Drop incremental cache after updating a klib set Enabled Partial Linkage may replace entire IR expressions with a stub during klib linking and loading. This may break the incremental cache dependency graph. Dropping incremental cache files after updating klibs (e.g. update klib version) covers almost all cases which could not be tracked due to a broken dependency graph. This patch could be reverted after fixing KT-57347 --- .../kotlin/ir/backend/js/ic/CacheUpdater.kt | 22 ++++++++++++++++--- .../incremental/AbstractInvalidationTest.kt | 11 ++-------- .../lib1/module.info | 1 + .../lib1/module.info | 4 ++-- .../main/module.info | 4 ++-- .../companionProperties/lib2/module.info | 1 + .../genericInlineFunctions/lib1/module.info | 2 +- .../genericInlineFunctions/main/module.info | 2 +- .../moveFilesBetweenModules/lib1/module.info | 1 + .../moveFilesBetweenModules/lib2/module.info | 1 + .../invalidation/removeFile/lib1/module.info | 1 + .../invalidation/removeFile/lib2/module.info | 1 + .../removeUnusedFile/lib1/module.info | 1 + .../invalidation/renameFile/lib1/module.info | 1 + .../renameModule/lib1-new/module.info | 4 +--- .../renameModule/lib1/module.info | 4 +--- .../renameModule/main/module.info | 4 ++-- .../splitJoinModule/lib2/module.info | 4 ++-- .../splitJoinModule/main/module.info | 10 +++++++-- .../invalidation/splitJoinModule/project.info | 4 ++-- .../updateExports/lib2/module.info | 2 ++ .../lib2/module.info | 1 + 22 files changed, 54 insertions(+), 32 deletions(-) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index 52cadb73ffb..850bb171eba 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -118,12 +118,28 @@ class CacheUpdater( libs.filter { it.libraryFile.canonicalPath in friendPaths } } - private val incrementalCaches = libraryDependencies.keys.associate { lib -> + private val klibCacheDirs = libraryDependencies.keys.map { lib -> val libFile = KotlinLibraryFile(lib) val file = File(libFile.path) val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX) - val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash") - libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir) + File(cacheRootDir, "${file.name}.$pathHash") + } + + init { + // Enabled Partial Linkage may replace entire IR expressions with a stub during klib linking and loading. + // This may break the incremental cache dependency graph. + // Dropping incremental cache files after updating klibs (e.g. update klib version) + // covers almost all cases which could not be tracked due to a broken dependency graph. + // TODO: This code could be removed after fixing KT-57347 + val oldKlibCaches = cacheRootDir.listFiles { file: File -> file.isDirectory }?.mapTo(hashSetOf()) { it.name } ?: emptySet() + val newKlibCaches = klibCacheDirs.mapTo(hashSetOf()) { it.name } + if (oldKlibCaches != newKlibCaches) { + cacheRootDir.deleteRecursively() + } + } + + private val incrementalCaches = libraryDependencies.keys.zip(klibCacheDirs).associate { (lib, libraryCacheDir) -> + KotlinLibraryFile(lib) to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir) } private val removedIncrementalCaches = buildList { diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index c6011bee165..3e9efbd36a0 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -161,15 +161,8 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend val moduleSourceDir = File(sourceDir, module) val moduleInfo = moduleInfos[module] ?: error("No module info found for $module") val moduleStep = moduleInfo.steps.getValue(projStepId) - val deletedFiles = mutableSetOf() for (modification in moduleStep.modifications) { - modification.execute(moduleTestDir, moduleSourceDir) { deletedFiles.add(it.name) } - } - - val expectedFileStats = moduleStep.expectedFileStats.toMutableMap() - if (deletedFiles.isNotEmpty()) { - val removedFiles = expectedFileStats[DirtyFileState.REMOVED_FILE.str] ?: emptySet() - expectedFileStats[DirtyFileState.REMOVED_FILE.str] = removedFiles + deletedFiles + modification.execute(moduleTestDir, moduleSourceDir) {} } val outputKlibFile = resolveModuleArtifact(module, buildDir) @@ -196,7 +189,7 @@ abstract class AbstractInvalidationTest(private val targetBackend: TargetBackend module.safeModuleName, outputKlibFile.canonicalPath, friends.map { it.canonicalPath }, - expectedFileStats, + moduleStep.expectedFileStats, dtsFile?.readText() ) } diff --git a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentFile/lib1/module.info b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentFile/lib1/module.info index f9ab121586a..3e4d90d4428 100644 --- a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentFile/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentFile/lib1/module.info @@ -25,5 +25,6 @@ STEP 6: modifications: D : l1b.kt removed inverse depends: l1a.kt + removed file: l1b.kt STEP 7: updated exports: l1a.kt diff --git a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/lib1/module.info b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/lib1/module.info index e4dfcb28c24..c1523f2c0b5 100644 --- a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/lib1/module.info @@ -1,11 +1,11 @@ STEP 0: added file: l1a.kt, l1b.kt STEP 1: - updated exports: l1b.kt, l1a.kt + added file: l1b.kt, l1a.kt STEP 2: updated exports: l1a.kt STEP 3: - removed inverse depends: l1b.kt, l1a.kt + added file: l1b.kt, l1a.kt STEP 4: modifications: U : l1a.4.kt -> l1a.kt diff --git a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/main/module.info b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/main/module.info index 96b32437e2f..82578c5e076 100644 --- a/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/main/module.info +++ b/js/js.translator/testData/incremental/invalidation/addUpdateRemoveDependentModule/main/module.info @@ -7,7 +7,7 @@ STEP 1: dependencies: lib1, lib2 modifications: U : m.1.kt -> m.kt - modified ir: m.kt + added file: m.kt STEP 2: dependencies: lib1, lib2 updated imports: m.kt @@ -15,6 +15,6 @@ STEP 3: dependencies: lib1 modifications: U : m.0.kt -> m.kt - modified ir: m.kt + added file: m.kt STEP 4: dependencies: lib1 diff --git a/js/js.translator/testData/incremental/invalidation/companionProperties/lib2/module.info b/js/js.translator/testData/incremental/invalidation/companionProperties/lib2/module.info index 67465f0208a..a37beeb1105 100644 --- a/js/js.translator/testData/incremental/invalidation/companionProperties/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/companionProperties/lib2/module.info @@ -42,6 +42,7 @@ STEP 7: D : l2pair.kt D : l2string.kt D : l2func.kt + removed file: l2pair.kt, l2string.kt, l2func.kt STEP 8: dependencies: lib1 updated imports: l2int.kt diff --git a/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/lib1/module.info b/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/lib1/module.info index dec40c07058..d9bd08e4415 100644 --- a/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/lib1/module.info @@ -33,7 +33,7 @@ STEP 7: STEP 8: modifications: U : l1.0.kt -> l1.kt - modified ir: l1.kt + added file: l1.kt STEP 9: modifications: U : l1.1.kt -> l1.kt diff --git a/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/main/module.info b/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/main/module.info index 6fc0dbfd7a1..ac71e23e291 100644 --- a/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/main/module.info +++ b/js/js.translator/testData/incremental/invalidation/genericInlineFunctions/main/module.info @@ -14,7 +14,7 @@ STEP 8: dependencies: lib1, libProxy modifications: U : m.proxy.kt -> m.kt - modified ir: m.kt + added file: m.kt STEP 9..13: dependencies: lib1, libProxy updated imports: m.kt diff --git a/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib1/module.info b/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib1/module.info index e22d3ecff6f..37e34cd7de0 100644 --- a/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib1/module.info @@ -3,6 +3,7 @@ STEP 0: STEP 1: modifications: D : l12.kt + removed file: l12.kt STEP 2: modifications: U : l12_n.2.kt -> l12_n.kt diff --git a/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib2/module.info b/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib2/module.info index 07eadf09ad4..77922bc3b91 100644 --- a/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/moveFilesBetweenModules/lib2/module.info @@ -12,3 +12,4 @@ STEP 2: modifications: D : l22.kt removed direct depends: l21.kt + removed file: l22.kt diff --git a/js/js.translator/testData/incremental/invalidation/removeFile/lib1/module.info b/js/js.translator/testData/incremental/invalidation/removeFile/lib1/module.info index da49635626b..ffb5cc91962 100644 --- a/js/js.translator/testData/incremental/invalidation/removeFile/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/removeFile/lib1/module.info @@ -3,3 +3,4 @@ STEP 0: STEP 1: modifications: D : l1_1.kt + removed file: l1_1.kt diff --git a/js/js.translator/testData/incremental/invalidation/removeFile/lib2/module.info b/js/js.translator/testData/incremental/invalidation/removeFile/lib2/module.info index 6f226a866d5..bfdff44b819 100644 --- a/js/js.translator/testData/incremental/invalidation/removeFile/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/removeFile/lib2/module.info @@ -5,3 +5,4 @@ STEP 1: dependencies: lib1 modifications: D : l2_1.kt + removed file: l2_1.kt diff --git a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info index 7f1029d67a0..53ef88e0e4a 100644 --- a/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/removeUnusedFile/lib1/module.info @@ -3,4 +3,5 @@ STEP 0: STEP 1: modifications: D : l1_1.kt + removed file: l1_1.kt STEP 2: diff --git a/js/js.translator/testData/incremental/invalidation/renameFile/lib1/module.info b/js/js.translator/testData/incremental/invalidation/renameFile/lib1/module.info index e5601abb703..bc0ff8f1b1e 100644 --- a/js/js.translator/testData/incremental/invalidation/renameFile/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/renameFile/lib1/module.info @@ -5,3 +5,4 @@ STEP 1: D : l1.kt U : l1.kt -> l1_new.kt added file: l1_new.kt + removed file: l1.kt diff --git a/js/js.translator/testData/incremental/invalidation/renameModule/lib1-new/module.info b/js/js.translator/testData/incremental/invalidation/renameModule/lib1-new/module.info index aa15d5b7cbf..0155ab2fe7a 100644 --- a/js/js.translator/testData/incremental/invalidation/renameModule/lib1-new/module.info +++ b/js/js.translator/testData/incremental/invalidation/renameModule/lib1-new/module.info @@ -7,6 +7,4 @@ STEP 2: modifications: U : l1.2.kt -> l1.kt modified ir: l1.kt -STEP 3: - removed file: l1.kt -STEP 4..5: +STEP 3..5: diff --git a/js/js.translator/testData/incremental/invalidation/renameModule/lib1/module.info b/js/js.translator/testData/incremental/invalidation/renameModule/lib1/module.info index 62f0f2e4c94..202996b9447 100644 --- a/js/js.translator/testData/incremental/invalidation/renameModule/lib1/module.info +++ b/js/js.translator/testData/incremental/invalidation/renameModule/lib1/module.info @@ -2,9 +2,7 @@ STEP 0: modifications: U : l1.0.kt -> l1.kt added file: l1.kt -STEP 1: - removed file: l1.kt -STEP 2: +STEP 1..2: STEP 3: modifications: U : l1.3.kt -> l1.kt diff --git a/js/js.translator/testData/incremental/invalidation/renameModule/main/module.info b/js/js.translator/testData/incremental/invalidation/renameModule/main/module.info index 0f7fd2be5e4..393d05936da 100644 --- a/js/js.translator/testData/incremental/invalidation/renameModule/main/module.info +++ b/js/js.translator/testData/incremental/invalidation/renameModule/main/module.info @@ -3,13 +3,13 @@ STEP 0: added file: m.kt STEP 1: dependencies: lib1-new - removed direct depends: m.kt + added file: m.kt STEP 2: dependencies: lib1-new updated imports: m.kt STEP 3: dependencies: lib1 - removed direct depends: m.kt + added file: m.kt STEP 4..5: dependencies: lib1 updated imports: m.kt diff --git a/js/js.translator/testData/incremental/invalidation/splitJoinModule/lib2/module.info b/js/js.translator/testData/incremental/invalidation/splitJoinModule/lib2/module.info index 37df220edf2..9c25ce5580d 100644 --- a/js/js.translator/testData/incremental/invalidation/splitJoinModule/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/splitJoinModule/lib2/module.info @@ -5,12 +5,12 @@ STEP 0: added file: l2.kt STEP 1: dependencies: lib1-a, lib1-b, lib1-c - removed direct depends: l2.kt + added file: l2.kt STEP 2..3: dependencies: lib1-a, lib1-b, lib1-c updated imports: l2.kt STEP 4: dependencies: lib1 - removed direct depends: l2.kt + added file: l2.kt STEP 5: dependencies: lib1 diff --git a/js/js.translator/testData/incremental/invalidation/splitJoinModule/main/module.info b/js/js.translator/testData/incremental/invalidation/splitJoinModule/main/module.info index bfa04d76033..7690a26dfc3 100644 --- a/js/js.translator/testData/incremental/invalidation/splitJoinModule/main/module.info +++ b/js/js.translator/testData/incremental/invalidation/splitJoinModule/main/module.info @@ -1,7 +1,13 @@ STEP 0: dependencies: lib1, lib2 added file: m.kt -STEP 1..3: +STEP 1: dependencies: lib1-a, lib1-b, lib1-c, lib2 -STEP 4..5: + added file: m.kt +STEP 2..3: + dependencies: lib1-a, lib1-b, lib1-c, lib2 +STEP 4: + dependencies: lib1, lib2 + added file: m.kt +STEP 5: dependencies: lib1, lib2 diff --git a/js/js.translator/testData/incremental/invalidation/splitJoinModule/project.info b/js/js.translator/testData/incremental/invalidation/splitJoinModule/project.info index e6e487ff080..40b9a393f89 100644 --- a/js/js.translator/testData/incremental/invalidation/splitJoinModule/project.info +++ b/js/js.translator/testData/incremental/invalidation/splitJoinModule/project.info @@ -5,13 +5,13 @@ STEP 0: dirty js: lib1, lib2, main STEP 1: libs: lib1-a, lib1-b, lib1-c, lib2, main - dirty js: lib1-a, lib1-b, lib1-c, lib2 + dirty js: lib1-a, lib1-b, lib1-c, main, lib2 STEP 2..3: libs: lib1-a, lib1-b, lib1-c, lib2, main dirty js: lib1-b, lib2 STEP 4: libs: lib1, lib2, main - dirty js: lib1, lib2 + dirty js: lib1, lib2, main STEP 5: libs: lib1, lib2, main dirty js: lib1 diff --git a/js/js.translator/testData/incremental/invalidation/updateExports/lib2/module.info b/js/js.translator/testData/incremental/invalidation/updateExports/lib2/module.info index 1af73547025..74187f9d595 100644 --- a/js/js.translator/testData/incremental/invalidation/updateExports/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/updateExports/lib2/module.info @@ -32,6 +32,7 @@ STEP 5: U : l2b.5.kt -> l2b.kt D : l2a.kt modified ir: l2b.kt + removed file: l2a.kt STEP 6: dependencies: lib1 modifications: @@ -43,6 +44,7 @@ STEP 7: U : l2a.7.kt -> l2a.kt D : l2b.kt added file: l2a.kt + removed file: l2b.kt STEP 8: dependencies: lib1 modifications: diff --git a/js/js.translator/testData/incremental/invalidation/updateExportsAndInlineImports/lib2/module.info b/js/js.translator/testData/incremental/invalidation/updateExportsAndInlineImports/lib2/module.info index 1f5ff1a3872..200bd3484b6 100644 --- a/js/js.translator/testData/incremental/invalidation/updateExportsAndInlineImports/lib2/module.info +++ b/js/js.translator/testData/incremental/invalidation/updateExportsAndInlineImports/lib2/module.info @@ -26,6 +26,7 @@ STEP 5: U : l2b.5.kt -> l2b.kt D : l2a.kt modified ir: l2b.kt + removed file: l2a.kt STEP 6: dependencies: lib1 updated imports: l2b.kt