From b719865c25aede19f8afaba4cf19e72c730ab4b3 Mon Sep 17 00:00:00 2001 From: Roman Artemev Date: Mon, 13 Dec 2021 17:18:09 +0300 Subject: [PATCH] [JS IR IC] Implement fast path invalidation check Compute library md5 hash and check it first with cached one. If hashes are equal it means that cache is up-to-date and no additional checks are needed - store library hashes (flat + trans) into cache info file - change invalidator return value --- .../jetbrains/kotlin/cli/js/K2JsIrCompiler.kt | 11 +- .../kotlin/ir/backend/js/ic/fileUtil.kt | 15 +-- .../kotlin/ir/backend/js/ic/invalidation.kt | 118 ++++++++++++++---- .../kotlin/ir/backend/js/ic/CacheAccessors.kt | 9 +- .../incremental/AbstractInvalidationTest.kt | 24 ++-- .../test/utils/JsIrIncrementalDataProvider.kt | 3 +- 6 files changed, 132 insertions(+), 48 deletions(-) diff --git a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt index 9033f344964..e929bdffab6 100644 --- a/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt +++ b/compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt @@ -208,7 +208,7 @@ class K2JsIrCompiler : CLICompiler() { val start = System.currentTimeMillis() - val updated = actualizeCacheForModule( + val updateStatus = actualizeCacheForModule( includes, outputFilePath, configurationJs, @@ -218,13 +218,10 @@ class K2JsIrCompiler : CLICompiler() { ::buildCacheForModuleFiles ) - if (updated) { - messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms") + if (updateStatus.upToDate) { + messageCollector.report(INFO, "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms") } else { - messageCollector.report( - INFO, - "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms" - ) + messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms") } return OK } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt index 7517b9f83f7..edbe482b793 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt @@ -45,7 +45,7 @@ fun buildCache( if (!forceClean) { val oldCacheInfo = CacheInfo.load(cachePath) - if (oldCacheInfo != null && md5 == oldCacheInfo.md5) return false + if (oldCacheInfo != null && md5 == oldCacheInfo.flatHash) return false } val icDir = File(cachePath) @@ -57,7 +57,7 @@ fun buildCache( icData.serializedIcData.writeTo(File(cachePath)) - CacheInfo(cachePath, mainModule.libPath, md5).save() + CacheInfo(cachePath, mainModule.libPath, md5, 0UL).save() return true } @@ -117,18 +117,19 @@ fun checkCaches( if (c.libPath !in allLibs) error("Missing library: ${c.libPath}") result[c.libPath] = File(c.path).readIcData() - md5[c.libPath] = c.md5 + md5[c.libPath] = c.flatHash } return IcCacheInfo(result, md5) } // TODO md5 hash -data class CacheInfo(val path: String, val libPath: String, val md5: ULong) { +data class CacheInfo(val path: String, val libPath: String, var flatHash: ULong, var transHash: ULong) { fun save() { PrintWriter(File(File(path), "info")).use { it.println(libPath) - it.println(md5.toString(16)) + it.println(flatHash.toString(16)) + it.println(transHash.toString(16)) } } @@ -138,9 +139,9 @@ data class CacheInfo(val path: String, val libPath: String, val md5: ULong) { if (!info.exists()) return null - val (libPath, md5) = info.readLines() + val (libPath, flatHash, transHash) = info.readLines() - return CacheInfo(path, libPath, md5.toULong(16)) + return CacheInfo(path, libPath, flatHash.toULong(16), transHash.toULong(16)) } } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt index 2d02c932948..cf5b8d83dde 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt @@ -36,6 +36,7 @@ import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl import org.jetbrains.kotlin.storage.LockBasedStorageManager import java.io.File +import java.security.MessageDigest import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile private fun KotlinLibrary.fingerprint(fileIndex: Int): Hash { @@ -142,7 +143,7 @@ private fun KotlinLibrary.filesAndSigReaders(): List): MutableMap { +private fun loadCacheInfo(cachePaths: Collection): MutableMap { val caches = cachePaths.map { CacheInfo.load(it) ?: error("Cannot load IC cache from $it") } - val result = mutableMapOf() - return caches.associateTo(result) { it.libPath.toCanonicalPath() to it.path } + val result = mutableMapOf() + return caches.associateByTo(result) { it.libPath.toCanonicalPath() } } private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Collection): Map { @@ -304,6 +305,76 @@ fun interface CacheExecutor { ) } +private fun File.md5(additional: Iterable = emptyList()): ULong { + val md5 = MessageDigest.getInstance("MD5") + + for (ul in additional) { + md5.update(ul.toLong().toByteArray()) + } + + fun File.process(prefix: String = "") { + if (isDirectory) { + this.listFiles()!!.sortedBy { it.name }.forEach { + md5.update((prefix + it.name).toByteArray()) + it.process(prefix + it.name + "/") + } + } else { + md5.update(readBytes()) + } + } + + this.process() + + val d = md5.digest() + + return ((d[0].toULong() and 0xFFUL) + or ((d[1].toULong() and 0xFFUL) shl 8) + or ((d[2].toULong() and 0xFFUL) shl 16) + or ((d[3].toULong() and 0xFFUL) shl 24) + or ((d[4].toULong() and 0xFFUL) shl 32) + or ((d[5].toULong() and 0xFFUL) shl 40) + or ((d[6].toULong() and 0xFFUL) shl 48) + or ((d[7].toULong() and 0xFFUL) shl 56) + ) +} + +private fun checkLibrariesHash( + libraries: Map, + dependencyGraph: Map>, + icCacheMap: Map, + modulePath: ModulePath +): Boolean { + val currentLib = libraries[modulePath] ?: error("1") + val currentCache = icCacheMap[modulePath] ?: error("2") + + val flatHash = File(modulePath).md5() + + val dependencies = dependencyGraph[currentLib] ?: error("3") + + var transHash = flatHash + + for (dep in dependencies) { + val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("4") + transHash += depCache.transHash + } + + if (currentCache.transHash != transHash) { + currentCache.flatHash = flatHash + currentCache.transHash = transHash + return false + } + + return true +} + +enum class CacheUpdateStatus(val upToDate: Boolean) { + DIRTY(upToDate = false), + NO_DIRTY_FILES(upToDate = true), + FAST_PATH(upToDate = true) + +} + +// Returns true if caches up-to-date fun actualizeCacheForModule( moduleName: String, cachePath: String, @@ -312,41 +383,46 @@ fun actualizeCacheForModule( icCachePaths: Collection, irFactory: IrFactory, executor: CacheExecutor -): Boolean { - val icCacheMap: Map = loadCacheInfo(icCachePaths).also { - it[moduleName.toCanonicalPath()] = cachePath +): CacheUpdateStatus { + val modulePath = moduleName.toCanonicalPath() + val cacheInfo = CacheInfo.load(cachePath) ?: CacheInfo(cachePath, modulePath, 0UL, 0UL) + val icCacheMap: Map = loadCacheInfo(icCachePaths).also { + it[modulePath] = cacheInfo } val libraries: Map = loadLibraries(compilerConfiguration, dependencies) - - val persistentCacheProviders = icCacheMap.map { (lib, cache) -> - libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache) - }.toMap() - val nameToKotlinLibrary: Map = libraries.values.associateBy { it.moduleName } - val dependencyGraph = libraries.values.associateWith { it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName -> nameToKotlinLibrary[depName] ?: error("No Library found for $depName") } } + if (checkLibrariesHash(libraries, dependencyGraph, icCacheMap, modulePath)) { + return CacheUpdateStatus.FAST_PATH // up-to-date + } + + val persistentCacheProviders = icCacheMap.map { (lib, cache) -> + libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache.path) + }.toMap() + val currentModule = libraries[moduleName.toCanonicalPath()] ?: error("No loaded library found for path $moduleName") val persistentCacheConsumer = createCacheConsumer(cachePath) - return actualizeCacheForModule(currentModule, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor) + return actualizeCacheForModule(currentModule, cacheInfo, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor) } private fun actualizeCacheForModule( library: KotlinLibrary, + libraryInfo: CacheInfo, configuration: CompilerConfiguration, dependencyGraph: Map>, persistentCacheProviders: Map, persistentCacheConsumer: PersistentCacheConsumer, irFactory: IrFactory, cacheExecutor: CacheExecutor -): Boolean { +): CacheUpdateStatus { // 1. Invalidate val dependencies = dependencyGraph[library]!! @@ -393,7 +469,7 @@ private fun actualizeCacheForModule( fileFingerPrints ) - if (dirtySet.isEmpty()) return true // up-to-date + if (dirtySet.isEmpty()) return CacheUpdateStatus.NO_DIRTY_FILES // up-to-date // 2. Build @@ -431,7 +507,7 @@ private fun actualizeCacheForModule( val deserializers = dirtySet.associateWith { currentModuleDeserializer.signatureDeserializerForFile(it).signatureToIndexMapping() } buildCacheForModule( - library.libraryFile.canonicalPath, + libraryInfo, configuration, currentIrModule, jsIrLinker, @@ -443,7 +519,7 @@ private fun actualizeCacheForModule( fileFingerPrints, cacheExecutor ) - return false // invalidated and re-built + return CacheUpdateStatus.DIRTY // invalidated and re-built } fun rebuildCacheForDirtyFiles( @@ -525,10 +601,10 @@ fun buildCacheForModuleFiles( fun loadModuleCaches(icCachePaths: Collection): Map { - val icCacheMap: Map = loadCacheInfo(icCachePaths) + val icCacheMap: Map = loadCacheInfo(icCachePaths) return icCacheMap.entries.associate { (lib, cache) -> - val provider = createCacheProvider(cache) + val provider = createCacheProvider(cache.path) val files = provider.filePaths() lib to ModuleCache(lib, files.associate { f -> f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f)) diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt index cecf5936a5d..63959bb567d 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt @@ -203,7 +203,7 @@ interface PersistentCacheConsumer { fun commitSourceMap(path: String, mapData: ByteArray) fun invalidateForFile(path: String) - fun commitLibraryPath(libraryPath: String) + fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) companion object { val EMPTY = object : PersistentCacheConsumer { @@ -234,7 +234,7 @@ interface PersistentCacheConsumer { } - override fun commitLibraryPath(libraryPath: String) { + override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) { } @@ -333,7 +333,7 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac commitByteArrayToCacheFile(path, fileSourceMap, mapData) } - override fun commitLibraryPath(libraryPath: String) { + override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) { val infoFile = File(File(cachePath), "info") if (infoFile.exists()) { infoFile.delete() @@ -342,7 +342,8 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac PrintWriter(infoFile).use { it.println(libraryPath) - it.println("0") + it.println(flatHash.toString(16)) + it.println(transHash.toString(16)) } } } diff --git a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt index 77ed57e103f..28e37ff6a73 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt @@ -17,6 +17,7 @@ import org.jetbrains.kotlin.codegen.* import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.ir.backend.js.generateKLib +import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdateStatus import org.jetbrains.kotlin.ir.backend.js.ic.PersistentCacheConsumer import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCacheForModule import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker @@ -145,7 +146,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { val moduleCacheDir = resolveModuleCache(module, buildDir) - buildCachesAndCheck(moduleStep.id, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles) + buildCachesAndCheck(moduleStep, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles) } } } @@ -156,7 +157,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { } private fun buildCachesAndCheck( - stepId: Int, + moduleStep: ModuleInfo.ModuleStep, configuration: CompilerConfiguration, sourceDir: File, moduleKlibFile: File, @@ -176,17 +177,20 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { exportedDeclarations: Set, mainArguments: List?, ) { - val actualDirtyFiles = invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath } + val actualDirtyFiles = + invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath } val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath } - JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) { "For module $moduleKlibFile at step $stepId" } + JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) { + "For module $moduleKlibFile at step ${moduleStep.id}" + } } val dependenciesPaths = mutableListOf() dependencies.mapTo(dependenciesPaths) { it.canonicalPath } dependenciesPaths.add(moduleKlibFile.canonicalPath) - val upToDate = actualizeCacheForModule( + val updateStatus = actualizeCacheForModule( moduleKlibFile.canonicalPath, moduleCacheDir.canonicalPath, configuration, @@ -196,9 +200,15 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() { ::dirtyFilesChecker ) - JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), upToDate) { + if (StepDirectives.FAST_PATH_UPDATE in moduleStep.directives) { + JUnit4Assertions.assertEquals(CacheUpdateStatus.FAST_PATH, updateStatus) { + "Cache has to be checked by fast path, instead it $updateStatus" + } + } + + JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), updateStatus.upToDate) { val filePaths = expectedDirtyFiles.joinToString(",", "[", "]") - "Up to date is not expected for module $moduleKlibFile at step $stepId. Expected dirtyFiles are $filePaths" + "Up to date is not expected for module $moduleKlibFile at step ${moduleStep.id}. Expected dirtyFiles are $filePaths" } } diff --git a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt index b74f2b8085c..2bfaf255590 100644 --- a/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt +++ b/js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt @@ -9,7 +9,6 @@ import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController import org.jetbrains.kotlin.ir.backend.js.ic.* import org.jetbrains.kotlin.ir.backend.js.moduleName -import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner @@ -115,7 +114,7 @@ class TestModuleCache(val moduleName: String, val files: MutableMap