[JS IR IC] Implement fast path invalidation check

Compute library md5 hash and check it first with cached one. If hashes
are equal it means that cache is up-to-date and no additional checks are
needed

 - store library hashes (flat + trans) into cache info file
 - change invalidator return value

compiler/cli/cli-js/src/org/jetbrains/kotlin/cli/js/K2JsIrCompiler.kt

@@ -208,7 +208,7 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
 
             val start = System.currentTimeMillis()
 
-            val updated = actualizeCacheForModule(
+            val updateStatus = actualizeCacheForModule(
                 includes,
                 outputFilePath,
                 configurationJs,
@@ -218,13 +218,10 @@ class K2JsIrCompiler : CLICompiler<K2JSCompilerArguments>() {
                 ::buildCacheForModuleFiles
             )
 
-            if (updated) {
+            if (updateStatus.upToDate) {
-                messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms")
+                messageCollector.report(INFO, "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms")
             } else {
-                messageCollector.report(
+                messageCollector.report(INFO, "IC per-file cache building duration: ${System.currentTimeMillis() - start}ms")
-                    INFO,
-                    "IC per-file cache up-to-date check duration: ${System.currentTimeMillis() - start}ms"
-                )
             }
             return OK
         }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/fileUtil.kt

@@ -45,7 +45,7 @@ fun buildCache(
 
     if (!forceClean) {
         val oldCacheInfo = CacheInfo.load(cachePath)
-        if (oldCacheInfo != null && md5 == oldCacheInfo.md5) return false
+        if (oldCacheInfo != null && md5 == oldCacheInfo.flatHash) return false
     }
 
     val icDir = File(cachePath)
@@ -57,7 +57,7 @@ fun buildCache(
 
     icData.serializedIcData.writeTo(File(cachePath))
 
-    CacheInfo(cachePath, mainModule.libPath, md5).save()
+    CacheInfo(cachePath, mainModule.libPath, md5, 0UL).save()
 
     return true
 }
@@ -117,18 +117,19 @@ fun checkCaches(
         if (c.libPath !in allLibs) error("Missing library: ${c.libPath}")
 
         result[c.libPath] = File(c.path).readIcData()
-        md5[c.libPath] = c.md5
+        md5[c.libPath] = c.flatHash
     }
 
     return IcCacheInfo(result, md5)
 }
 
 // TODO md5 hash
-data class CacheInfo(val path: String, val libPath: String, val md5: ULong) {
+data class CacheInfo(val path: String, val libPath: String, var flatHash: ULong, var transHash: ULong) {
     fun save() {
         PrintWriter(File(File(path), "info")).use {
             it.println(libPath)
-            it.println(md5.toString(16))
+            it.println(flatHash.toString(16))
+            it.println(transHash.toString(16))
         }
     }
 
@@ -138,9 +139,9 @@ data class CacheInfo(val path: String, val libPath: String, val md5: ULong) {
 
             if (!info.exists()) return null
 
-            val (libPath, md5) = info.readLines()
+            val (libPath, flatHash, transHash) = info.readLines()
 
-            return CacheInfo(path, libPath, md5.toULong(16))
+            return CacheInfo(path, libPath, flatHash.toULong(16), transHash.toULong(16))
         }
     }
 }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/invalidation.kt

@@ -36,6 +36,7 @@ import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import org.jetbrains.kotlin.psi2ir.generators.TypeTranslatorImpl
 import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import java.io.File
+import java.security.MessageDigest
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
 
 private fun KotlinLibrary.fingerprint(fileIndex: Int): Hash {
@@ -142,7 +143,7 @@ private fun KotlinLibrary.filesAndSigReaders(): List<Pair<String, IdSignatureDes
 }
 
 private fun buildCacheForModule(
-    libraryPath: String,
+    libraryInfo: CacheInfo,
     configuration: CompilerConfiguration,
     irModule: IrModuleFragment,
     deserializer: JsIrLinker,
@@ -195,7 +196,7 @@ private fun buildCacheForModule(
 
     cacheExecutor.execute(irModule, dependencies, deserializer, configuration, dirtyFiles, cacheConsumer, emptySet(), null) // TODO: main arguments?
 
-    cacheConsumer.commitLibraryPath(libraryPath)
+    cacheConsumer.commitLibraryPath(libraryInfo.libPath.toCanonicalPath(), libraryInfo.flatHash, libraryInfo.transHash)
 }
 
 private fun loadModules(
@@ -268,10 +269,10 @@ private fun createCacheConsumer(path: String): PersistentCacheConsumer {
     return PersistentCacheConsumerImpl(path)
 }
 
-private fun loadCacheInfo(cachePaths: Collection<String>): MutableMap<ModulePath, String> {
+private fun loadCacheInfo(cachePaths: Collection<String>): MutableMap<ModulePath, CacheInfo> {
     val caches = cachePaths.map { CacheInfo.load(it) ?: error("Cannot load IC cache from $it") }
-    val result = mutableMapOf<ModulePath, String>()
+    val result = mutableMapOf<ModulePath, CacheInfo>()
-    return caches.associateTo(result) { it.libPath.toCanonicalPath() to it.path }
+    return caches.associateByTo(result) { it.libPath.toCanonicalPath() }
 }
 
 private fun loadLibraries(configuration: CompilerConfiguration, dependencies: Collection<String>): Map<ModulePath, KotlinLibrary> {
@@ -304,6 +305,76 @@ fun interface CacheExecutor {
     )
 }
 
+private fun File.md5(additional: Iterable<ULong> = emptyList()): ULong {
+    val md5 = MessageDigest.getInstance("MD5")
+
+    for (ul in additional) {
+        md5.update(ul.toLong().toByteArray())
+    }
+
+    fun File.process(prefix: String = "") {
+        if (isDirectory) {
+            this.listFiles()!!.sortedBy { it.name }.forEach {
+                md5.update((prefix + it.name).toByteArray())
+                it.process(prefix + it.name + "/")
+            }
+        } else {
+            md5.update(readBytes())
+        }
+    }
+
+    this.process()
+
+    val d = md5.digest()
+
+    return ((d[0].toULong() and 0xFFUL)
+            or ((d[1].toULong() and 0xFFUL) shl 8)
+            or ((d[2].toULong() and 0xFFUL) shl 16)
+            or ((d[3].toULong() and 0xFFUL) shl 24)
+            or ((d[4].toULong() and 0xFFUL) shl 32)
+            or ((d[5].toULong() and 0xFFUL) shl 40)
+            or ((d[6].toULong() and 0xFFUL) shl 48)
+            or ((d[7].toULong() and 0xFFUL) shl 56)
+            )
+}
+
+private fun checkLibrariesHash(
+    libraries: Map<ModulePath, KotlinLibrary>,
+    dependencyGraph: Map<KotlinLibrary, List<KotlinLibrary>>,
+    icCacheMap: Map<ModulePath, CacheInfo>,
+    modulePath: ModulePath
+): Boolean {
+    val currentLib = libraries[modulePath] ?: error("1")
+    val currentCache = icCacheMap[modulePath] ?: error("2")
+
+    val flatHash = File(modulePath).md5()
+
+    val dependencies = dependencyGraph[currentLib] ?: error("3")
+
+    var transHash = flatHash
+
+    for (dep in dependencies) {
+        val depCache = icCacheMap[dep.libraryFile.canonicalPath] ?: error("4")
+        transHash += depCache.transHash
+    }
+
+    if (currentCache.transHash != transHash) {
+        currentCache.flatHash = flatHash
+        currentCache.transHash = transHash
+        return false
+    }
+
+    return true
+}
+
+enum class CacheUpdateStatus(val upToDate: Boolean) {
+    DIRTY(upToDate = false),
+    NO_DIRTY_FILES(upToDate = true),
+    FAST_PATH(upToDate = true)
+
+}
+
+// Returns true if caches up-to-date
 fun actualizeCacheForModule(
     moduleName: String,
     cachePath: String,
@@ -312,41 +383,46 @@ fun actualizeCacheForModule(
     icCachePaths: Collection<String>,
     irFactory: IrFactory,
     executor: CacheExecutor
-): Boolean {
+): CacheUpdateStatus {
-    val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths).also {
+    val modulePath = moduleName.toCanonicalPath()
-        it[moduleName.toCanonicalPath()] = cachePath
+    val cacheInfo = CacheInfo.load(cachePath) ?: CacheInfo(cachePath, modulePath, 0UL, 0UL)
+    val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths).also {
+        it[modulePath] = cacheInfo
     }
 
     val libraries: Map<ModulePath, KotlinLibrary> = loadLibraries(compilerConfiguration, dependencies)
-
-    val persistentCacheProviders = icCacheMap.map { (lib, cache) ->
-        libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache)
-    }.toMap()
-
     val nameToKotlinLibrary: Map<ModuleName, KotlinLibrary> = libraries.values.associateBy { it.moduleName }
-
     val dependencyGraph = libraries.values.associateWith {
         it.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).map { depName ->
             nameToKotlinLibrary[depName] ?: error("No Library found for $depName")
         }
     }
 
+    if (checkLibrariesHash(libraries, dependencyGraph, icCacheMap, modulePath)) {
+        return CacheUpdateStatus.FAST_PATH // up-to-date
+    }
+
+    val persistentCacheProviders = icCacheMap.map { (lib, cache) ->
+        libraries[lib.toCanonicalPath()]!! to createCacheProvider(cache.path)
+    }.toMap()
+
     val currentModule = libraries[moduleName.toCanonicalPath()] ?: error("No loaded library found for path $moduleName")
     val persistentCacheConsumer = createCacheConsumer(cachePath)
 
-    return actualizeCacheForModule(currentModule, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor)
+    return actualizeCacheForModule(currentModule, cacheInfo, compilerConfiguration, dependencyGraph, persistentCacheProviders, persistentCacheConsumer, irFactory, executor)
 }
 
 
 private fun actualizeCacheForModule(
     library: KotlinLibrary,
+    libraryInfo: CacheInfo,
     configuration: CompilerConfiguration,
     dependencyGraph: Map<KotlinLibrary, Collection<KotlinLibrary>>,
     persistentCacheProviders: Map<KotlinLibrary, PersistentCacheProvider>,
     persistentCacheConsumer: PersistentCacheConsumer,
     irFactory: IrFactory,
     cacheExecutor: CacheExecutor
-): Boolean {
+): CacheUpdateStatus {
     // 1. Invalidate
     val dependencies = dependencyGraph[library]!!
 
@@ -393,7 +469,7 @@ private fun actualizeCacheForModule(
         fileFingerPrints
     )
 
-    if (dirtySet.isEmpty()) return true // up-to-date
+    if (dirtySet.isEmpty()) return CacheUpdateStatus.NO_DIRTY_FILES // up-to-date
 
     // 2. Build
 
@@ -431,7 +507,7 @@ private fun actualizeCacheForModule(
     val deserializers = dirtySet.associateWith { currentModuleDeserializer.signatureDeserializerForFile(it).signatureToIndexMapping() }
 
     buildCacheForModule(
-        library.libraryFile.canonicalPath,
+        libraryInfo,
         configuration,
         currentIrModule,
         jsIrLinker,
@@ -443,7 +519,7 @@ private fun actualizeCacheForModule(
         fileFingerPrints,
         cacheExecutor
     )
-    return false // invalidated and re-built
+    return CacheUpdateStatus.DIRTY // invalidated and re-built
 }
 
 fun rebuildCacheForDirtyFiles(
@@ -525,10 +601,10 @@ fun buildCacheForModuleFiles(
 
 
 fun loadModuleCaches(icCachePaths: Collection<String>): Map<String, ModuleCache> {
-    val icCacheMap: Map<ModulePath, String> = loadCacheInfo(icCachePaths)
+    val icCacheMap: Map<ModulePath, CacheInfo> = loadCacheInfo(icCachePaths)
 
     return icCacheMap.entries.associate { (lib, cache) ->
-        val provider = createCacheProvider(cache)
+        val provider = createCacheProvider(cache.path)
         val files = provider.filePaths()
         lib to ModuleCache(lib, files.associate { f ->
             f to FileCache(f, provider.binaryAst(f), provider.dts(f), provider.sourceMap(f))

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheAccessors.kt

@@ -203,7 +203,7 @@ interface PersistentCacheConsumer {
     fun commitSourceMap(path: String, mapData: ByteArray)
     fun invalidateForFile(path: String)
 
-    fun commitLibraryPath(libraryPath: String)
+    fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong)
 
     companion object {
         val EMPTY = object : PersistentCacheConsumer {
@@ -234,7 +234,7 @@ interface PersistentCacheConsumer {
 
             }
 
-            override fun commitLibraryPath(libraryPath: String) {
+            override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
 
             }
 
@@ -333,7 +333,7 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac
         commitByteArrayToCacheFile(path, fileSourceMap, mapData)
     }
 
-    override fun commitLibraryPath(libraryPath: String) {
+    override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
         val infoFile = File(File(cachePath), "info")
         if (infoFile.exists()) {
             infoFile.delete()
@@ -342,7 +342,8 @@ class PersistentCacheConsumerImpl(private val cachePath: String) : PersistentCac
 
         PrintWriter(infoFile).use {
             it.println(libraryPath)
-            it.println("0")
+            it.println(flatHash.toString(16))
+            it.println(transHash.toString(16))
         }
     }
 }

js/js.tests/test/org/jetbrains/kotlin/incremental/AbstractInvalidationTest.kt

@@ -17,6 +17,7 @@ import org.jetbrains.kotlin.codegen.*
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.backend.js.generateKLib
+import org.jetbrains.kotlin.ir.backend.js.ic.CacheUpdateStatus
 import org.jetbrains.kotlin.ir.backend.js.ic.PersistentCacheConsumer
 import org.jetbrains.kotlin.ir.backend.js.ic.actualizeCacheForModule
 import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
@@ -145,7 +146,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
 
                 val moduleCacheDir = resolveModuleCache(module, buildDir)
 
-                buildCachesAndCheck(moduleStep.id, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles)
+                buildCachesAndCheck(moduleStep, configuration, moduleSourceDir, outputKlibFile, moduleCacheDir, dependencies, icCaches, dirtyFiles)
             }
         }
     }
@@ -156,7 +157,7 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
     }
 
     private fun buildCachesAndCheck(
-        stepId: Int,
+        moduleStep: ModuleInfo.ModuleStep,
         configuration: CompilerConfiguration,
         sourceDir: File,
         moduleKlibFile: File,
@@ -176,17 +177,20 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
             exportedDeclarations: Set<FqName>,
             mainArguments: List<String>?,
         ) {
-            val actualDirtyFiles = invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath }
+            val actualDirtyFiles =
+                invalidatedDirtyFiles?.map { File(it).canonicalPath } ?: sourceDir.filteredKtFiles().map { it.canonicalPath }
             val expectedDirtyFilesCanonical = expectedDirtyFiles.map { it.canonicalPath }
 
-            JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) { "For module $moduleKlibFile at step $stepId" }
+            JUnit4Assertions.assertSameElements(expectedDirtyFilesCanonical, actualDirtyFiles) {
+                "For module $moduleKlibFile at step ${moduleStep.id}"
+            }
         }
 
         val dependenciesPaths = mutableListOf<String>()
         dependencies.mapTo(dependenciesPaths) { it.canonicalPath }
         dependenciesPaths.add(moduleKlibFile.canonicalPath)
 
-        val upToDate = actualizeCacheForModule(
+        val updateStatus = actualizeCacheForModule(
             moduleKlibFile.canonicalPath,
             moduleCacheDir.canonicalPath,
             configuration,
@@ -196,9 +200,15 @@ abstract class AbstractInvalidationTest : KotlinTestWithEnvironment() {
             ::dirtyFilesChecker
         )
 
-        JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), upToDate) {
+        if (StepDirectives.FAST_PATH_UPDATE in moduleStep.directives) {
+            JUnit4Assertions.assertEquals(CacheUpdateStatus.FAST_PATH, updateStatus) {
+                "Cache has to be checked by fast path, instead it $updateStatus"
+            }
+        }
+
+        JUnit4Assertions.assertEquals(expectedDirtyFiles.isEmpty(), updateStatus.upToDate) {
             val filePaths = expectedDirtyFiles.joinToString(",", "[", "]")
-            "Up to date is not expected for module $moduleKlibFile at step $stepId. Expected dirtyFiles are $filePaths"
+            "Up to date is not expected for module $moduleKlibFile at step ${moduleStep.id}. Expected dirtyFiles are $filePaths"
         }
     }
 

js/js.tests/test/org/jetbrains/kotlin/js/test/utils/JsIrIncrementalDataProvider.kt

@@ -9,7 +9,6 @@ import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.backend.js.WholeWorldStageController
 import org.jetbrains.kotlin.ir.backend.js.ic.*
 import org.jetbrains.kotlin.ir.backend.js.moduleName
-import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImpl
 import org.jetbrains.kotlin.ir.declarations.impl.IrFactoryImplForJsIC
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.js.test.handlers.JsBoxRunner
@@ -115,7 +114,7 @@ class TestModuleCache(val moduleName: String, val files: MutableMap<String, File
                 files.remove(path)
             }
 
-            override fun commitLibraryPath(libraryPath: String) {
+            override fun commitLibraryPath(libraryPath: String, flatHash: ULong, transHash: ULong) {
 
             }
         }