[JS IR IC] Use fingerprints form klib && use 128 bytes hash

Optimization: instead of calculating fingerprints manually each time the patch allows using precalculated fingerprints from the klib. Optimization: share one md5 digest for all IC infrastructure.
2022-12-15 14:53:35 +01:00
parent c502cae437
commit dd268d67ff
6 changed files with 121 additions and 118 deletions
@@ -4,6 +4,7 @@

 package org.jetbrains.kotlin.ir.backend.js.ic

+import org.jetbrains.kotlin.backend.common.serialization.cityHash64
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity
@@ -63,7 +64,12 @@ class CacheUpdater(

    private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath)

-    private val cacheRootDir = File(cacheDir, "version.${compilerConfiguration.configHashForIC()}")
+    private val icHasher = ICHasher()
+
+    private val cacheRootDir = run {
+        val configHash = icHasher.calculateConfigHash(compilerConfiguration)
+        File(cacheDir, "version.${configHash.hash.lowBytes.toString(Character.MAX_RADIX)}")
+    }

    fun getDirtyFileLastStats(): KotlinSourceFileMap<EnumSet<DirtyFileState>> = dirtyFileStats

@@ -77,7 +83,7 @@ class CacheUpdater(
    }

    private inner class CacheUpdaterInternal {
-        val signatureHashCalculator = IdSignatureHashCalculator()
+        val signatureHashCalculator = IdSignatureHashCalculator(icHasher)

        // libraries in topological order: [stdlib, ..., main]
        val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") {
@@ -114,7 +120,8 @@ class CacheUpdater(
        private val incrementalCaches = libraryDependencies.keys.associate { lib ->
            val libFile = KotlinLibraryFile(lib)
            val file = File(libFile.path)
-            val libraryCacheDir = File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}")
+            val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX)
+            val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash")
            libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir)
        }

@@ -734,6 +741,7 @@ fun rebuildCacheForDirtyFiles(

    val libFile = KotlinLibraryFile(library)
    val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLoadedLibraryHeader(library).sourceFileFingerprints.keys
+
    val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata })

    val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, emptyList(), irFactory)
@@ -5,42 +5,48 @@

 package org.jetbrains.kotlin.ir.backend.js.ic

+import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.languageVersionSettings
 import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CrossModuleReferences
 import org.jetbrains.kotlin.ir.IrElement
 import org.jetbrains.kotlin.ir.declarations.IrAnnotationContainer
 import org.jetbrains.kotlin.ir.declarations.IrFunction
-import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
 import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer
 import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.util.DumpIrTreeVisitor
 import org.jetbrains.kotlin.js.config.JSConfigurationKeys
-import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.impl.buffer
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
-import java.io.File
 import java.security.MessageDigest

+internal fun Hash128Bits.toProtoStream(out: CodedOutputStream) {
+    out.writeFixed64NoTag(lowBytes.toLong())
+    out.writeFixed64NoTag(highBytes.toLong())
+}
+
+internal fun readHash128BitsFromProtoStream(input: CodedInputStream): Hash128Bits {
+    val lowBytes = input.readFixed64().toULong()
+    val highBytes = input.readFixed64().toULong()
+    return Hash128Bits(lowBytes, highBytes)
+}
+
@JvmInline
-value class ICHash(private val value: ULong = 0UL) {
-    fun combineWith(other: ICHash) = ICHash(value xor (other.value + 0x9e3779b97f4a7c15UL + (value shl 12) + (value shr 4)))
-
-    override fun toString() = value.toString(16)
-
-    fun toProtoStream(out: CodedOutputStream) = out.writeFixed64NoTag(value.toLong())
+value class ICHash(val hash: Hash128Bits = Hash128Bits()) {
+    fun toProtoStream(out: CodedOutputStream) = hash.toProtoStream(out)

    companion object {
-        fun fromProtoStream(input: CodedInputStream) = ICHash(input.readFixed64().toULong())
+        fun fromProtoStream(input: CodedInputStream) = ICHash(readHash128BitsFromProtoStream(input))
    }
 }

 private class HashCalculatorForIC {
-    private val md5 = MessageDigest.getInstance("MD5")
+    private val md5Digest = MessageDigest.getInstance("MD5")

-    fun update(data: ByteArray) = md5.update(data)
+    fun update(data: ByteArray) = md5Digest.update(data)

-    fun update(data: Int) = (0..3).forEach { md5.update((data shr (it * 8)).toByte()) }
+    fun update(data: Int) = (0..3).forEach { md5Digest.update((data shr (it * 8)).toByte()) }

    fun update(data: String) {
        update(data.length)
@@ -68,87 +74,64 @@ private class HashCalculatorForIC {
        collection.forEach { f(it) }
    }

-    private fun bytesToULong(d: ByteArray, offset: Int): ULong {
-        var hash = 0UL
-        repeat(8) {
-            hash = hash or ((d[it + offset].toULong() and 0xFFUL) shl (it * 8))
-        }
-        return hash
-    }
-
    fun finalize(): ICHash {
-        val d = md5.digest()
-        return ICHash(bytesToULong(d, 0)).combineWith(ICHash(bytesToULong(d, 8)))
+        val hashBytes = md5Digest.digest()
+        md5Digest.reset()
+        return hashBytes.buffer.asLongBuffer().let { longBuffer ->
+            ICHash(Hash128Bits(longBuffer[0].toULong(), longBuffer[1].toULong()))
+        }
    }
 }

-internal fun File.fileHashForIC(): ICHash {
-    val md5 = HashCalculatorForIC()
-    fun File.process(prefix: String = "") {
-        if (isDirectory) {
-            this.listFiles()!!.sortedBy { it.name }.forEach {
-                md5.update(prefix + it.name)
-                it.process(prefix + it.name + "/")
+internal class ICHasher {
+    private val hashCalculator = HashCalculatorForIC()
+
+    fun calculateConfigHash(config: CompilerConfiguration): ICHash {
+        val importantSettings = listOf(
+            JSConfigurationKeys.GENERATE_DTS,
+            JSConfigurationKeys.MODULE_KIND,
+            JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION
+        )
+        hashCalculator.updateForEach(importantSettings) { key ->
+            hashCalculator.update(key.toString())
+            hashCalculator.update(config.get(key).toString())
+        }
+
+        hashCalculator.update(config.languageVersionSettings.toString())
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrFunctionHash(function: IrFunction): ICHash {
+        hashCalculator.update(function)
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrAnnotationContainerHash(container: IrAnnotationContainer): ICHash {
+        hashCalculator.updateAnnotationContainer(container)
+        return hashCalculator.finalize()
+    }
+
+    fun calculateIrSymbolHash(symbol: IrSymbol): ICHash {
+        hashCalculator.update(symbol.toString())
+        // symbol rendering prints very little information about type parameters
+        // TODO may be it make sense to update rendering?
+        (symbol.owner as? IrTypeParametersContainer)?.let { typeParameters ->
+            hashCalculator.updateForEach(typeParameters.typeParameters) { typeParameter ->
+                hashCalculator.update(typeParameter.symbol.toString())
            }
-        } else {
-            md5.update(readBytes())
        }
+        (symbol.owner as? IrFunction)?.let { irFunction ->
+            hashCalculator.updateForEach(irFunction.valueParameters) { functionParam ->
+                // symbol rendering doesn't print default params information
+                // it is important to understand if default params were added or removed
+                hashCalculator.update(functionParam.defaultValue?.let { 1 } ?: 0)
+            }
+        }
+        (symbol.owner as? IrAnnotationContainer)?.let(hashCalculator::updateAnnotationContainer)
+        return hashCalculator.finalize()
    }
-    this.process()
-    return md5.finalize()
 }

-internal fun CompilerConfiguration.configHashForIC() = HashCalculatorForIC().apply {
-    val importantSettings = listOf(
-        JSConfigurationKeys.GENERATE_DTS,
-        JSConfigurationKeys.MODULE_KIND,
-        JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION
-    )
-    updateForEach(importantSettings) { key ->
-        update(key.toString())
-        update(get(key).toString())
-    }
-
-    update(languageVersionSettings.toString())
-}.finalize()
-
-internal fun IrFunction.irFunctionHashForIC() = HashCalculatorForIC().also {
-    it.update(this)
-}.finalize()
-
-internal fun IrAnnotationContainer.irAnnotationContainerHashForIC() = HashCalculatorForIC().also {
-    it.updateAnnotationContainer(this)
-}.finalize()
-
-internal fun IrSymbol.irSymbolHashForIC() = HashCalculatorForIC().also {
-    it.update(toString())
-    // symbol rendering prints very little information about type parameters
-    // TODO may be it make sense to update rendering?
-    (owner as? IrTypeParametersContainer)?.let { typeParameters ->
-        it.updateForEach(typeParameters.typeParameters) { typeParameter ->
-            it.update(typeParameter.symbol.toString())
-        }
-    }
-    (owner as? IrFunction)?.let { irFunction ->
-        it.updateForEach(irFunction.valueParameters) { functionParam ->
-            // symbol rendering doesn't print default params information
-            // it is important to understand if default params were added or removed
-            it.update(functionParam.defaultValue?.let { 1 } ?: 0)
-        }
-    }
-    (owner as? IrAnnotationContainer)?.let(it::updateAnnotationContainer)
-}.finalize()
-
-internal fun String.stringHashForIC() = HashCalculatorForIC().also { it.update(this) }.finalize()
-
-internal fun KotlinLibrary.fingerprint(fileIndex: Int) = HashCalculatorForIC().apply {
-    update(types(fileIndex))
-    update(signatures(fileIndex))
-    update(strings(fileIndex))
-    update(declarations(fileIndex))
-    update(bodies(fileIndex))
-}.finalize()
-
 internal fun CrossModuleReferences.crossModuleReferencesHashForIC() = HashCalculatorForIC().apply {
    update(moduleKind.ordinal)

@@ -19,7 +19,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
 import org.jetbrains.kotlin.ir.visitors.acceptVoid
 import org.jetbrains.kotlin.utils.addToStdlib.ifTrue

-internal class IdSignatureHashCalculator {
+internal class IdSignatureHashCalculator(private val icHasher: ICHasher) {
    private val idSignatureSources = hashMapOf<IdSignature, IdSignatureSource>()
    private val idSignatureHashes = hashMapOf<IdSignature, ICHash>()

@@ -29,17 +29,19 @@ internal class IdSignatureHashCalculator {
    private val inlineFunctionDepends = hashMapOf<IrFunction, LinkedHashSet<IrFunction>>()

    private val IrFile.annotationsHash: ICHash
-        get() = fileAnnotationHashes.getOrPut(this, ::irAnnotationContainerHashForIC)
+        get() = fileAnnotationHashes.getOrPut(this) {
+            icHasher.calculateIrAnnotationContainerHash(this)
+        }

    private val IrFunction.inlineFunctionFlatHash: ICHash
        get() = inlineFunctionFlatHashes.getOrPut(this) {
            val flatHash = if (isFakeOverride && this is IrSimpleFunction) {
-                resolveFakeOverride()?.irFunctionHashForIC()
+                resolveFakeOverride()?.let { icHasher.calculateIrFunctionHash(it) }
                    ?: icError("can not resolve fake override for ${render()}")
            } else {
-                irFunctionHashForIC()
+                icHasher.calculateIrFunctionHash(this)
            }
-            symbol.calculateSymbolHash().combineWith(flatHash)
+            ICHash(symbol.calculateSymbolHash().hash.combineWith(flatHash.hash))
        }

    private val IrFunction.inlineDepends: Collection<IrFunction>
@@ -85,7 +87,7 @@ internal class IdSignatureHashCalculator {
        }

        val fileAnnotationsHash = srcIrFile?.annotationsHash ?: ICHash()
-        return fileAnnotationsHash.combineWith(irSymbolHashForIC())
+        return ICHash(fileAnnotationsHash.hash.combineWith(icHasher.calculateIrSymbolHash(this).hash))
    }

    private fun IrFunction.calculateInlineFunctionTransitiveHash(): ICHash {
@@ -97,7 +99,7 @@ internal class IdSignatureHashCalculator {
            newDependsStack.removeLast().inlineDepends.forEach { inlineFunction ->
                if (transitiveDepends.add(inlineFunction)) {
                    newDependsStack += inlineFunction
-                    transitiveHash = transitiveHash.combineWith(inlineFunction.inlineFunctionFlatHash)
+                    transitiveHash = ICHash(transitiveHash.hash.combineWith(inlineFunction.inlineFunctionFlatHash.hash))
                }
            }
        }
@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.ir.backend.js.ic

 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
+import org.jetbrains.kotlin.backend.common.serialization.cityHash64
+import org.jetbrains.kotlin.ir.backend.js.FingerprintHash
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream
@@ -40,31 +42,31 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca

    private class CacheHeader(
        val libraryFile: KotlinLibraryFile,
-        val libraryFingerprint: ICHash?,
-        val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>
+        val libraryFingerprint: FingerprintHash?,
+        val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash>
    ) {
        constructor(library: KotlinLibraryHeader) : this(library.libraryFile, library.libraryFingerprint, library.sourceFileFingerprints)

        fun toProtoStream(out: CodedOutputStream) {
            libraryFile.toProtoStream(out)

-            libraryFingerprint?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile)
+            libraryFingerprint?.hash?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile)

            out.writeInt32NoTag(sourceFileFingerprints.size)
            for ((srcFile, fingerprint) in sourceFileFingerprints) {
                srcFile.toProtoStream(out)
-                fingerprint.toProtoStream(out)
+                fingerprint.hash.toProtoStream(out)
            }
        }

        companion object {
            fun fromProtoStream(input: CodedInputStream): CacheHeader {
                val libraryFile = KotlinLibraryFile.fromProtoStream(input)
-                val oldLibraryFingerprint = ICHash.fromProtoStream(input)
+                val oldLibraryFingerprint = FingerprintHash(readHash128BitsFromProtoStream(input))

                val sourceFileFingerprints = buildMapUntil(input.readInt32()) {
                    val file = KotlinSourceFile.fromProtoStream(input)
-                    put(file, ICHash.fromProtoStream(input))
+                    put(file, FingerprintHash(readHash128BitsFromProtoStream(input)))
                }
                return CacheHeader(libraryFile, oldLibraryFingerprint, sourceFileFingerprints)
            }
@@ -76,7 +78,10 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca
        override val directDependencies: KotlinSourceFileMap<Map<IdSignature, ICHash>>,
    ) : KotlinSourceFileMetadata()

-    private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix")
+    private fun KotlinSourceFile.getCacheFile(suffix: String): File {
+        val pathHash = path.cityHash64().toULong().toString(Character.MAX_RADIX)
+        return File(cacheDir, "${File(path).name}.$pathHash.$suffix")
+    }

    fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map<KotlinSourceFile, Map<IdSignature, Int>>): IncrementalCacheArtifact {
        val klibSrcFiles = if (cacheHeaderShouldBeUpdated) {
@@ -5,12 +5,9 @@

 package org.jetbrains.kotlin.ir.backend.js.ic

-import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource
-import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes
-import org.jetbrains.kotlin.backend.common.serialization.codedInputStream
+import org.jetbrains.kotlin.backend.common.serialization.*
 import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile
-import org.jetbrains.kotlin.ir.backend.js.jsOutputName
+import org.jetbrains.kotlin.ir.backend.js.*
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
 import java.io.File
@@ -18,18 +15,26 @@ import java.io.File
 internal interface KotlinLibraryHeader {
    val libraryFile: KotlinLibraryFile

-    val libraryFingerprint: ICHash?
+    val libraryFingerprint: FingerprintHash?

    val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer>
-    val sourceFileFingerprints: Map<KotlinSourceFile, ICHash>
+    val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash>

    val jsOutputName: String?
 }

 internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : KotlinLibraryHeader {
+    private fun parseFingerprintsFromManifest(): Map<KotlinSourceFile, FingerprintHash>? {
+        val manifestFingerprints = library.serializedIrFileFingerprints?.takeIf { it.size == sourceFiles.size } ?: return null
+        return sourceFiles.withIndex().associate { it.value to manifestFingerprints[it.index].fileFingerprint }
+    }
+
    override val libraryFile: KotlinLibraryFile = KotlinLibraryFile(library)

-    override val libraryFingerprint: ICHash by lazy { File(libraryFile.path).fileHashForIC() }
+    override val libraryFingerprint: FingerprintHash by lazy {
+        val serializedKlib = library.serializedKlibFingerprint ?: SerializedKlibFingerprint(File(libraryFile.path))
+        serializedKlib.klibFingerprint
+    }

    override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> by lazy {
        buildMapUntil(sourceFiles.size) {
@@ -47,9 +52,9 @@ internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : K
        }
    }

-    override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> by lazy {
-        buildMapUntil(sourceFiles.size) {
-            put(sourceFiles[it], library.fingerprint(it))
+    override val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash> by lazy {
+        parseFingerprintsFromManifest() ?: buildMapUntil(sourceFiles.size) {
+            put(sourceFiles[it], SerializedIrFileFingerprint(library, it).fileFingerprint)
        }
    }

@@ -69,10 +74,10 @@ internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : Kotli
    override val libraryFile: KotlinLibraryFile
        get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}")

-    override val libraryFingerprint: ICHash? get() = null
+    override val libraryFingerprint: FingerprintHash? get() = null

    override val sourceFileDeserializers: Map<KotlinSourceFile, IdSignatureDeserializer> get() = emptyMap()
-    override val sourceFileFingerprints: Map<KotlinSourceFile, ICHash> get() = emptyMap()
+    override val sourceFileFingerprints: Map<KotlinSourceFile, FingerprintHash> get() = emptyMap()

    override val jsOutputName: String? get() = null
 }
@@ -15,7 +15,7 @@ import java.io.File
@JvmInline
 value class FingerprintHash(val hash: Hash128Bits) {
    override fun toString(): String {
-        return "${hash.highBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.lowBytes.toString(Character.MAX_RADIX)}"
+        return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}"
    }

    companion object {
@@ -23,7 +23,7 @@ value class FingerprintHash(val hash: Hash128Bits) {

        internal fun fromString(s: String): FingerprintHash? {
            val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) }
-            return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(it[0], it[1])) }
+            return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) }
        }
    }
 }
@@ -77,7 +77,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
            if (isDirectory) {
                listFiles()!!.sortedBy { it.name }.forEach { f ->
                    val filePrefix = "$prefix${f.name}/"
-                    combinedHash = calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray()))
+                    combinedHash = f.calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray()))
                }
            } else {
                forEachBlock { buffer, bufferSize ->