[JS IR IC] Use fingerprints form klib && use 128 bytes hash

Optimization: instead of calculating fingerprints manually each time
 the patch allows using precalculated fingerprints from the klib.

 Optimization: share one md5 digest for all IC infrastructure.

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt

@@ -4,6 +4,7 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
+import org.jetbrains.kotlin.backend.common.serialization.cityHash64
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
@@ -63,7 +64,12 @@ class CacheUpdater(
 
     private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)
 
-    private val cacheRootDir = File(cacheDir, "version.${compilerConfiguration.configHashForIC()}")
+    private val icHasher = ICHasher()
+
+    private val cacheRootDir = run {
+        val configHash = icHasher.calculateConfigHash(compilerConfiguration)
+        File(cacheDir, "version.${configHash.hash.lowBytes.toString(Character.MAX_RADIX)}")
+    }
 
     fun getDirtyFileLastStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats
 
@@ -77,7 +83,7 @@ class CacheUpdater(
     }
 
     private inner class CacheUpdaterInternal {
-        val signatureHashCalculator = IdSignatureHashCalculator()
+        val signatureHashCalculator = IdSignatureHashCalculator(icHasher)
 
         // libraries in topological order: [stdlib, ..., main]
         val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") {
@@ -114,7 +120,8 @@ class CacheUpdater(
         private val incrementalCaches = libraryDependencies.keys.associate { lib ->
             val libFile = KotlinLibraryFile(lib)
             val file = File(libFile.path)
-            val libraryCacheDir = File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}")
+            val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX)
+            val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash")
             libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
         }
 
@@ -734,6 +741,7 @@ fun rebuildCacheForDirtyFiles(
 
     val libFile = KotlinLibraryFile(library)
     val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLoadedLibraryHeader(library).sourceFileFingerprints.keys
+
     val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })
 
     val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, emptyList(), irFactory)

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/HashCalculatorForIC.kt

@@ -5,42 +5,48 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
+import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CrossModuleReferences
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrAnnotationContainer
 import org.jetbrains.kotlin.ir.declarations.IrFunction
-import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.util.DumpIrTreeVisitor
 import org.jetbrains.kotlin.js.config.JSConfigurationKeys
-import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.buffer
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
-import java.io.File
 import java.security.MessageDigest
 
+internal fun Hash128Bits.toProtoStream(out: CodedOutputStream) {
+    out.writeFixed64NoTag(lowBytes.toLong())
+    out.writeFixed64NoTag(highBytes.toLong())
+}
+
+internal fun readHash128BitsFromProtoStream(input: CodedInputStream): Hash128Bits {
+    val lowBytes = input.readFixed64().toULong()
+    val highBytes = input.readFixed64().toULong()
+    return Hash128Bits(lowBytes, highBytes)
+}
+
 @JvmInline
-value class ICHash(private val value: ULong = 0UL) {
+value class ICHash(val hash: Hash128Bits = Hash128Bits()) {
-    fun combineWith(other: ICHash) = ICHash(value xor (other.value + 0x9e3779b97f4a7c15UL + (value shl 12) + (value shr 4)))
+    fun toProtoStream(out: CodedOutputStream) = hash.toProtoStream(out)
-
-    override fun toString() = value.toString(16)
-
-    fun toProtoStream(out: CodedOutputStream) = out.writeFixed64NoTag(value.toLong())
 
     companion object {
-        fun fromProtoStream(input: CodedInputStream) = ICHash(input.readFixed64().toULong())
+        fun fromProtoStream(input: CodedInputStream) = ICHash(readHash128BitsFromProtoStream(input))
     }
 }
 
 private class HashCalculatorForIC {
-    private val md5 = MessageDigest.getInstance("MD5")
+    private val md5Digest = MessageDigest.getInstance("MD5")
 
-    fun update(data: ByteArray) = md5.update(data)
+    fun update(data: ByteArray) = md5Digest.update(data)
 
-    fun update(data: Int) = (0..3).forEach { md5.update((data shr (it * 8)).toByte()) }
+    fun update(data: Int) = (0..3).forEach { md5Digest.update((data shr (it * 8)).toByte()) }
 
     fun update(data: String) {
         update(data.length)
@@ -68,87 +74,64 @@ private class HashCalculatorForIC {
         collection.forEach { f(it) }
     }
 
-    private fun bytesToULong(d: ByteArray, offset: Int): ULong {
-        var hash = 0UL
-        repeat(8) {
-            hash = hash or ((d[it + offset].toULong() and 0xFFUL) shl (it * 8))
-        }
-        return hash
-    }
-
     fun finalize(): ICHash {
-        val d = md5.digest()
+        val hashBytes = md5Digest.digest()
-        return ICHash(bytesToULong(d, 0)).combineWith(ICHash(bytesToULong(d, 8)))
+        md5Digest.reset()
+        return hashBytes.buffer.asLongBuffer().let { longBuffer ->
+            ICHash(Hash128Bits(longBuffer[0].toULong(), longBuffer[1].toULong()))
+        }
     }
 }
 
-internal fun File.fileHashForIC(): ICHash {
+internal class ICHasher {
-    val md5 = HashCalculatorForIC()
+    private val hashCalculator = HashCalculatorForIC()
-    fun File.process(prefix: String = "") {
+
-        if (isDirectory) {
+    fun calculateConfigHash(config: CompilerConfiguration): ICHash {
-            this.listFiles()!!.sortedBy { it.name }.forEach {
+        val importantSettings = listOf(
-                md5.update(prefix + it.name)
+            JSConfigurationKeys.GENERATE_DTS,
-                it.process(prefix + it.name + "/")
+            JSConfigurationKeys.MODULE_KIND,
+            JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION
+        )
+        hashCalculator.updateForEach(importantSettings) { key ->
+            hashCalculator.update(key.toString())
+            hashCalculator.update(config.get(key).toString())
+        }
+
+        hashCalculator.update(config.languageVersionSettings.toString())
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrFunctionHash(function: IrFunction): ICHash {
+        hashCalculator.update(function)
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrAnnotationContainerHash(container: IrAnnotationContainer): ICHash {
+        hashCalculator.updateAnnotationContainer(container)
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrSymbolHash(symbol: IrSymbol): ICHash {
+        hashCalculator.update(symbol.toString())
+        // symbol rendering prints very little information about type parameters
+        // TODO may be it make sense to update rendering?
+        (symbol.owner as? IrTypeParametersContainer)?.let { typeParameters ->
+            hashCalculator.updateForEach(typeParameters.typeParameters) { typeParameter ->
+                hashCalculator.update(typeParameter.symbol.toString())
             }
-        } else {
-            md5.update(readBytes())
         }
+        (symbol.owner as? IrFunction)?.let { irFunction ->
+            hashCalculator.updateForEach(irFunction.valueParameters) { functionParam ->
+                // symbol rendering doesn't print default params information
+                // it is important to understand if default params were added or removed
+                hashCalculator.update(functionParam.defaultValue?.let { 1 } ?: 0)
+            }
+        }
+        (symbol.owner as? IrAnnotationContainer)?.let(hashCalculator::updateAnnotationContainer)
+        return hashCalculator.finalize()
     }
-    this.process()
-    return md5.finalize()
 }
 
-internal fun CompilerConfiguration.configHashForIC() = HashCalculatorForIC().apply {
-    val importantSettings = listOf(
-        JSConfigurationKeys.GENERATE_DTS,
-        JSConfigurationKeys.MODULE_KIND,
-        JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION
-    )
-    updateForEach(importantSettings) { key ->
-        update(key.toString())
-        update(get(key).toString())
-    }
-
-    update(languageVersionSettings.toString())
-}.finalize()
-
-internal fun IrFunction.irFunctionHashForIC() = HashCalculatorForIC().also {
-    it.update(this)
-}.finalize()
-
-internal fun IrAnnotationContainer.irAnnotationContainerHashForIC() = HashCalculatorForIC().also {
-    it.updateAnnotationContainer(this)
-}.finalize()
-
-internal fun IrSymbol.irSymbolHashForIC() = HashCalculatorForIC().also {
-    it.update(toString())
-    // symbol rendering prints very little information about type parameters
-    // TODO may be it make sense to update rendering?
-    (owner as? IrTypeParametersContainer)?.let { typeParameters ->
-        it.updateForEach(typeParameters.typeParameters) { typeParameter ->
-            it.update(typeParameter.symbol.toString())
-        }
-    }
-    (owner as? IrFunction)?.let { irFunction ->
-        it.updateForEach(irFunction.valueParameters) { functionParam ->
-            // symbol rendering doesn't print default params information
-            // it is important to understand if default params were added or removed
-            it.update(functionParam.defaultValue?.let { 1 } ?: 0)
-        }
-    }
-    (owner as? IrAnnotationContainer)?.let(it::updateAnnotationContainer)
-}.finalize()
-
-internal fun String.stringHashForIC() = HashCalculatorForIC().also { it.update(this) }.finalize()
-
-internal fun KotlinLibrary.fingerprint(fileIndex: Int) = HashCalculatorForIC().apply {
-    update(types(fileIndex))
-    update(signatures(fileIndex))
-    update(strings(fileIndex))
-    update(declarations(fileIndex))
-    update(bodies(fileIndex))
-}.finalize()
-
 internal fun CrossModuleReferences.crossModuleReferencesHashForIC() = HashCalculatorForIC().apply {
     update(moduleKind.ordinal)
 

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureHashCalculator.kt

@@ -19,7 +19,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.utils.addToStdlib.ifTrue
 
-internal class IdSignatureHashCalculator {
+internal class IdSignatureHashCalculator(private val icHasher: ICHasher) {
     private val idSignatureSources = hashMapOf<IdSignature, IdSignatureSource>()
     private val idSignatureHashes = hashMapOf<IdSignature, ICHash>()
 
@@ -29,17 +29,19 @@ internal class IdSignatureHashCalculator {
     private val inlineFunctionDepends = hashMapOf<IrFunction, LinkedHashSet<IrFunction>>()
 
     private val IrFile.annotationsHash: ICHash
-        get() = fileAnnotationHashes.getOrPut(this, ::irAnnotationContainerHashForIC)
+        get() = fileAnnotationHashes.getOrPut(this) {
+            icHasher.calculateIrAnnotationContainerHash(this)
+        }
 
     private val IrFunction.inlineFunctionFlatHash: ICHash
         get() = inlineFunctionFlatHashes.getOrPut(this) {
             val flatHash = if (isFakeOverride && this is IrSimpleFunction) {
-                resolveFakeOverride()?.irFunctionHashForIC()
+                resolveFakeOverride()?.let { icHasher.calculateIrFunctionHash(it) }
                     ?: icError("can not resolve fake override for ${render()}")
             } else {
-                irFunctionHashForIC()
+                icHasher.calculateIrFunctionHash(this)
             }
-            symbol.calculateSymbolHash().combineWith(flatHash)
+            ICHash(symbol.calculateSymbolHash().hash.combineWith(flatHash.hash))
         }
 
     private val IrFunction.inlineDepends: Collection<IrFunction>
@@ -85,7 +87,7 @@ internal class IdSignatureHashCalculator {
         }
 
         val fileAnnotationsHash = srcIrFile?.annotationsHash ?: ICHash()
-        return fileAnnotationsHash.combineWith(irSymbolHashForIC())
+        return ICHash(fileAnnotationsHash.hash.combineWith(icHasher.calculateIrSymbolHash(this).hash))
     }
 
     private fun IrFunction.calculateInlineFunctionTransitiveHash(): ICHash {
@@ -97,7 +99,7 @@ internal class IdSignatureHashCalculator {
             newDependsStack.removeLast().inlineDepends.forEach { inlineFunction ->
                 if (transitiveDepends.add(inlineFunction)) {
                     newDependsStack += inlineFunction
-                    transitiveHash = transitiveHash.combineWith(inlineFunction.inlineFunctionFlatHash)
+                    transitiveHash = ICHash(transitiveHash.hash.combineWith(inlineFunction.inlineFunctionFlatHash.hash))
                 }
             }
         }

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.ir.backend.js.ic
 
 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.backend.common.serialization.cityHash64
+import org.jetbrains.kotlin.ir.backend.js.FingerprintHash
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
@@ -40,31 +42,31 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
 
     private class CacheHeader(
         val libraryFile: KotlinLibraryFile,
-        val libraryFingerprint: ICHash?,
+        val libraryFingerprint: FingerprintHash?,
-        val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>
+        val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash>
     ) {
         constructor(library: KotlinLibraryHeader) : this(library.libraryFile, library.libraryFingerprint, library.sourceFileFingerprints)
 
         fun toProtoStream(out: CodedOutputStream) {
             libraryFile.toProtoStream(out)
 
-            libraryFingerprint?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile)
+            libraryFingerprint?.hash?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile)
 
             out.writeInt32NoTag(sourceFileFingerprints.size)
             for ((srcFile, fingerprint) in sourceFileFingerprints) {
                 srcFile.toProtoStream(out)
-                fingerprint.toProtoStream(out)
+                fingerprint.hash.toProtoStream(out)
             }
         }
 
         companion object {
             fun fromProtoStream(input: CodedInputStream): CacheHeader {
                 val libraryFile = KotlinLibraryFile.fromProtoStream(input)
-                val oldLibraryFingerprint = ICHash.fromProtoStream(input)
+                val oldLibraryFingerprint = FingerprintHash(readHash128BitsFromProtoStream(input))
 
                 val sourceFileFingerprints = buildMapUntil(input.readInt32()) {
                     val file = KotlinSourceFile.fromProtoStream(input)
-                    put(file, ICHash.fromProtoStream(input))
+                    put(file, FingerprintHash(readHash128BitsFromProtoStream(input)))
                 }
                 return CacheHeader(libraryFile, oldLibraryFingerprint, sourceFileFingerprints)
             }
@@ -76,7 +78,10 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
         override val directDependencies: KotlinSourceFileMap<Map<IdSignature, ICHash>>,
     ) : KotlinSourceFileMetadata()
 
-    private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix")
+    private fun KotlinSourceFile.getCacheFile(suffix: String): File {
+        val pathHash = path.cityHash64().toULong().toString(Character.MAX_RADIX)
+        return File(cacheDir, "${File(path).name}.$pathHash.$suffix")
+    }
 
     fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>): IncrementalCacheArtifact {
         val klibSrcFiles = if (cacheHeaderShouldBeUpdated) {

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt

@@ -5,12 +5,9 @@
 
 package org.jetbrains.kotlin.ir.backend.js.ic
 
-import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.backend.common.serialization.*
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
-import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
-import org.jetbrains.kotlin.ir.backend.js.jsOutputName
+import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import java.io.File
@@ -18,18 +15,26 @@ import java.io.File
 internal interface KotlinLibraryHeader {
     val libraryFile: KotlinLibraryFile
 
-    val libraryFingerprint: ICHash?
+    val libraryFingerprint: FingerprintHash?
 
     val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer>
-    val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>
+    val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash>
 
     val jsOutputName: String?
 }
 
 internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : KotlinLibraryHeader {
+    private fun parseFingerprintsFromManifest(): Map<KotlinSourceFile, FingerprintHash>? {
+        val manifestFingerprints = library.serializedIrFileFingerprints?.takeIf { it.size == sourceFiles.size } ?: return null
+        return sourceFiles.withIndex().associate { it.value to manifestFingerprints[it.index].fileFingerprint }
+    }
+
     override val libraryFile: KotlinLibraryFile = KotlinLibraryFile(library)
 
-    override val libraryFingerprint: ICHash by lazy { File(libraryFile.path).fileHashForIC() }
+    override val libraryFingerprint: FingerprintHash by lazy {
+        val serializedKlib = library.serializedKlibFingerprint ?: SerializedKlibFingerprint(File(libraryFile.path))
+        serializedKlib.klibFingerprint
+    }
 
     override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> by lazy {
         buildMapUntil(sourceFiles.size) {
@@ -47,9 +52,9 @@ internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : K
         }
     }
 
-    override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> by lazy {
+    override val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash> by lazy {
-        buildMapUntil(sourceFiles.size) {
+        parseFingerprintsFromManifest() ?: buildMapUntil(sourceFiles.size) {
-            put(sourceFiles[it], library.fingerprint(it))
+            put(sourceFiles[it], SerializedIrFileFingerprint(library, it).fileFingerprint)
         }
     }
 
@@ -69,10 +74,10 @@ internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : Kotli
     override val libraryFile: KotlinLibraryFile
         get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}")
 
-    override val libraryFingerprint: ICHash? get() = null
+    override val libraryFingerprint: FingerprintHash? get() = null
 
     override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> get() = emptyMap()
-    override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> get() = emptyMap()
+    override val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash> get() = emptyMap()
 
     override val jsOutputName: String? get() = null
 }

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt

@@ -15,7 +15,7 @@ import java.io.File
 @JvmInline
 value class FingerprintHash(val hash: Hash128Bits) {
     override fun toString(): String {
-        return "${hash.highBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.lowBytes.toString(Character.MAX_RADIX)}"
+        return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}"
     }
 
     companion object {
@@ -23,7 +23,7 @@ value class FingerprintHash(val hash: Hash128Bits) {
 
         internal fun fromString(s: String): FingerprintHash? {
             val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) }
-            return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(it[0], it[1])) }
+            return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) }
         }
     }
 }
@@ -77,7 +77,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
             if (isDirectory) {
                 listFiles()!!.sortedBy { it.name }.forEach { f ->
                     val filePrefix = "$prefix${f.name}/"
-                    combinedHash = calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray()))
+                    combinedHash = f.calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray()))
                 }
             } else {
                 forEachBlock { buffer, bufferSize ->