[JS IR] Write IrFile fingerprints to KotlinJs klib manifest

The file fingerprints are going to be used for a cache invalidation in JS IR incremental cache. It should reduce an incremental rebuild time in JS BE. CityHash128 is used as IrFile fingerprint.
2023-01-02 13:11:53 +01:00
parent 511e05adec
commit 3744fbb31b
4 changed files with 639 additions and 2 deletions
@@ -12,6 +12,7 @@ private val k0 = 0xc3a5c85c97cb3127U
 private val k1 = 0xb492b66fbe98f273U
 private val k2 = 0x9ae16a3b2f90404fU
 private val kMul = 0x9ddfea08eb382d69UL
+private val kGoldenRatio = 0x9e3779b97f4a7c15UL

 private fun toLongLE(b: ByteArray, i: Int): Long {
    return (b[i + 7].toLong() shl 56) +
@@ -214,3 +215,124 @@ public fun cityHash64(s: ByteArray, pos: Int = 0, len: Int = s.size): ULong {
@OptIn(ExperimentalUnsignedTypes::class)
 fun String.cityHash64(): Long =
    cityHash64(this.toByteArray()).toLong()
+
+data class Hash128Bits(val lowBytes: ULong = k0, val highBytes: ULong = k1) {
+    private infix fun ULong.combineHash(other: ULong) = other xor (this + kGoldenRatio + (other shl 12) + (other shr 4))
+
+    fun combineWith(other: Hash128Bits): Hash128Bits {
+        return Hash128Bits(highBytes combineHash other.lowBytes, lowBytes combineHash other.highBytes)
+    }
+}
+
+private fun cityMurmur(seed: Hash128Bits, s: ByteArray, pos: Int = 0, len: Int = s.size): Hash128Bits {
+    var pos = pos
+    var len = len
+
+    var (a, b) = seed
+    var c: ULong
+    var d: ULong
+
+    if (len <= 16) {
+        a = shiftMix(a * k1) * k1;
+        c = b * k1 + hashLen0to16(s, pos, len)
+        d = shiftMix(a + (if (len >= 8) fetch64(s, pos) else c))
+    } else {
+        c = hashLen16(fetch64(s, pos + len - 8) + k1, a)
+        d = hashLen16(b + len.toULong(), c + fetch64(s, pos + len - 16))
+        a += d
+        do {
+            a = a xor shiftMix(fetch64(s, pos) * k1) * k1
+            a *= k1
+            b = b xor a
+            c = c xor shiftMix(fetch64(s, pos + 8) * k1) * k1
+            c *= k1
+            d = d xor c
+            pos += 16
+            len -= 16
+        } while (len > 16)
+    }
+    a = hashLen16(a, c)
+    b = hashLen16(d, b)
+    return Hash128Bits(a xor b, hashLen16(b, a))
+}
+
+@OptIn(ExperimentalUnsignedTypes::class)
+fun cityHash128WithSeed(seed: Hash128Bits, s: ByteArray, pos: Int = 0, len: Int = s.size): Hash128Bits {
+    if (len < 128) {
+        return cityMurmur(seed, s, pos, len);
+    }
+
+    var pos = pos
+    var len = len
+
+    var v = ULongArray(2)
+    var w = ULongArray(2)
+
+    var (x, y) = seed
+
+    var z = len.toULong() * k1
+    v[0] = rotate(y xor k1, 49) * k1 + fetch64(s, pos)
+    v[1] = rotate(v[0], 42) * k1 + fetch64(s, pos + 8)
+    w[0] = rotate(y + z, 35) * k1 + x
+    w[1] = rotate(x + fetch64(s, pos + 88), 53) * k1
+
+    do {
+        x = rotate(x + y + v[0] + fetch64(s, pos + 8), 37) * k1
+        y = rotate(y + v[1] + fetch64(s, pos + 48), 42) * k1
+        x = x xor w[1]
+        y += v[0] + fetch64(s, pos + 40)
+        z = rotate(z + w[0], 33) * k1
+        v = weakHashLen32WithSeeds(s, pos, v[1] * k1, x + w[0])
+        w = weakHashLen32WithSeeds(s, pos + 32, z + w[1], y + fetch64(s, pos + 16))
+        run {
+            val swap = z
+            z = x
+            x = swap
+        }
+        pos += 64
+
+        x = rotate(x + y + v[0] + fetch64(s, pos + 8), 37) * k1
+        y = rotate(y + v[1] + fetch64(s, pos + 48), 42) * k1
+        x = x xor w[1]
+        y += v[0] + fetch64(s, pos + 40)
+        z = rotate(z + w[0], 33) * k1
+        v = weakHashLen32WithSeeds(s, pos, v[1] * k1, x + w[0])
+        w = weakHashLen32WithSeeds(s, pos + 32, z + w[1], y + fetch64(s, pos + 16))
+        run {
+            val swap = z
+            z = x
+            x = swap
+        }
+        pos += 64
+        len -= 128
+    } while (len >= 128)
+    x += rotate(v[0] + z, 49) * k0
+    y = y * k0 + rotate(w[1], 37)
+    z = z * k0 + rotate(w[0], 27)
+    w[0] *= 9UL
+    v[0] *= k0
+
+    var tailDone = 0
+    while (tailDone < len) {
+        tailDone += 32
+        y = rotate(x + y, 42) * k0 + v[1]
+        w[0] += fetch64(s, pos + len - tailDone + 16)
+        x = x * k0 + w[0]
+        z += w[1] + fetch64(s, pos + len - tailDone)
+        w[1] += v[0]
+        v = weakHashLen32WithSeeds(s, pos + len - tailDone, v[0] + z, v[1])
+        v[0] *= k0
+    }
+
+    x = hashLen16(x, v[0])
+    y = hashLen16(y + z, w[0])
+    return Hash128Bits(hashLen16(x + v[1], w[1]) + y, hashLen16(x + w[1], y + v[1]))
+}
+
+fun cityHash128(s: ByteArray, pos: Int = 0, len: Int = s.size): Hash128Bits {
+    return if (len >= 16) {
+        cityHash128WithSeed(Hash128Bits(fetch64(s, pos), fetch64(s, pos + 8) + k0), s, pos + 16, len - 16)
+    } else {
+        cityHash128WithSeed(Hash128Bits(k0, k1), s, pos, len)
+    }
+}
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2010-2023 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.ir.backend.js
+
+import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
+import org.jetbrains.kotlin.backend.common.serialization.cityHash128
+import org.jetbrains.kotlin.backend.common.serialization.cityHash128WithSeed
+import org.jetbrains.kotlin.library.KotlinLibrary
+import org.jetbrains.kotlin.library.SerializedIrFile
+import java.io.File
+
+@JvmInline
+value class FingerprintHash(val hash: Hash128Bits) {
+    override fun toString(): String {
+        return "${hash.highBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.lowBytes.toString(Character.MAX_RADIX)}"
+    }
+
+    companion object {
+        private const val HASH_SEPARATOR = "."
+
+        internal fun fromString(s: String): FingerprintHash? {
+            val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) }
+            return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(it[0], it[1])) }
+        }
+    }
+}
+
+@JvmInline
+value class SerializedIrFileFingerprint private constructor(val fileFingerprint: FingerprintHash) {
+    companion object {
+        internal fun fromString(s: String): SerializedIrFileFingerprint? {
+            return FingerprintHash.fromString(s)?.let { SerializedIrFileFingerprint(it) }
+        }
+
+        private fun calculateFileFingerprint(file: SerializedIrFile): FingerprintHash {
+            val fileDataHash = cityHash128(file.fileData)
+            val withTypesHash = cityHash128WithSeed(fileDataHash, file.types)
+            val withSignaturesHash = cityHash128WithSeed(withTypesHash, file.signatures)
+            val withStringsHash = cityHash128WithSeed(withSignaturesHash, file.strings)
+            val withBodiesHash = cityHash128WithSeed(withStringsHash, file.bodies)
+            return FingerprintHash(cityHash128WithSeed(withBodiesHash, file.declarations))
+        }
+
+        private fun calculateFileFingerprint(lib: KotlinLibrary, fileIndex: Int): FingerprintHash {
+            val fileDataHash = cityHash128(lib.file(fileIndex))
+            val withTypesHash = cityHash128WithSeed(fileDataHash, lib.types(fileIndex))
+            val withSignaturesHash = cityHash128WithSeed(withTypesHash, lib.signatures(fileIndex))
+            val withStringsHash = cityHash128WithSeed(withSignaturesHash, lib.strings(fileIndex))
+            val withBodiesHash = cityHash128WithSeed(withStringsHash, lib.bodies(fileIndex))
+            return FingerprintHash(cityHash128WithSeed(withBodiesHash, lib.declarations(fileIndex)))
+        }
+    }
+
+    constructor(file: SerializedIrFile) : this(calculateFileFingerprint(file))
+
+    constructor(lib: KotlinLibrary, fileIndex: Int) : this(calculateFileFingerprint(lib, fileIndex))
+}
+
+@JvmInline
+value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
+    companion object {
+        private fun List<SerializedIrFileFingerprint>.calculateKlibFingerprint(): FingerprintHash {
+            val combinedHash = fold(Hash128Bits(size.toULong())) { acc, x ->
+                acc.combineWith(x.fileFingerprint.hash)
+            }
+            return FingerprintHash(combinedHash)
+        }
+
+        // File.calculateKlibHash() and List<SerializedIrFileFingerprint>.calculateKlibFingerprint()
+        // give different hashes for same klibs, it is ok
+        private fun File.calculateKlibHash(prefix: String = "", seed: Hash128Bits = Hash128Bits()): Hash128Bits {
+            var combinedHash = seed
+
+            if (isDirectory) {
+                listFiles()!!.sortedBy { it.name }.forEach { f ->
+                    val filePrefix = "$prefix${f.name}/"
+                    combinedHash = calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray()))
+                }
+            } else {
+                forEachBlock { buffer, bufferSize ->
+                    combinedHash = cityHash128WithSeed(combinedHash, buffer, 0, bufferSize)
+                }
+            }
+
+            return combinedHash
+        }
+
+        internal fun fromString(s: String): SerializedKlibFingerprint? {
+            return FingerprintHash.fromString(s)?.let { SerializedKlibFingerprint(it) }
+        }
+    }
+
+    constructor(fileFingerprints: List<SerializedIrFileFingerprint>) : this(fileFingerprints.calculateKlibFingerprint())
+
+    constructor(klibFile: File) : this(FingerprintHash(klibFile.calculateKlibHash()))
+}
+
+private const val FILE_FINGERPRINTS_SEPARATOR = " "
+
+internal fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
+    return joinToString(FILE_FINGERPRINTS_SEPARATOR)
+}
+
+internal fun String.parseSerializedIrFileFingerprints(): List<SerializedIrFileFingerprint> {
+    return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString)
+}
@@ -87,6 +87,12 @@ val KotlinLibrary.isBuiltIns: Boolean
 val KotlinLibrary.jsOutputName: String?
    get() = manifestProperties.getProperty(KLIB_PROPERTY_JS_OUTPUT_NAME)

+val KotlinLibrary.serializedIrFileFingerprints: List<SerializedIrFileFingerprint>?
+    get() = manifestProperties.getProperty(KLIB_PROPERTY_SERIALIZED_IR_FILE_FINGERPRINTS)?.parseSerializedIrFileFingerprints()
+
+val KotlinLibrary.serializedKlibFingerprint: SerializedKlibFingerprint?
+    get() = manifestProperties.getProperty(KLIB_PROPERTY_SERIALIZED_KLIB_FINGERPRINT)?.let { SerializedKlibFingerprint.fromString(it) }
+
 private val CompilerConfiguration.metadataVersion
    get() = get(CommonConfigurationKeys.METADATA_VERSION) as? KlibMetadataVersion ?: KlibMetadataVersion.INSTANCE

@@ -336,7 +342,7 @@ fun getIrModuleInfoForSourceFiles(
        icData = null,
        friendModules = friendModules,
    )
-    val deserializedModuleFragmentsToLib = deserializeDependencies(allSortedDependencies, irLinker, null,null, mapping)
+    val deserializedModuleFragmentsToLib = deserializeDependencies(allSortedDependencies, irLinker, null, null, mapping)
    val deserializedModuleFragments = deserializedModuleFragmentsToLib.keys.toList()
    (irBuiltIns as IrBuiltInsOverDescriptors).functionFactory =
        IrDescriptorBasedFunctionFactory(
@@ -689,6 +695,10 @@ fun serializeModuleIntoKlib(
        if (containsErrorCode) {
            p.setProperty(KLIB_PROPERTY_CONTAINS_ERROR_CODE, "true")
        }
+
+        val fingerprints = fullSerializedIr.files.sortedBy { it.path }.map { SerializedIrFileFingerprint(it) }
+        p.setProperty(KLIB_PROPERTY_SERIALIZED_IR_FILE_FINGERPRINTS, fingerprints.joinIrFileFingerprints())
+        p.setProperty(KLIB_PROPERTY_SERIALIZED_KLIB_FINGERPRINT, SerializedKlibFingerprint(fingerprints).klibFingerprint.toString())
    }

    buildKotlinLibrary(
@@ -707,6 +717,8 @@ fun serializeModuleIntoKlib(
 }

 const val KLIB_PROPERTY_JS_OUTPUT_NAME = "jsOutputName"
+const val KLIB_PROPERTY_SERIALIZED_IR_FILE_FINGERPRINTS = "serializedIrFileFingerprints"
+const val KLIB_PROPERTY_SERIALIZED_KLIB_FINGERPRINT = "serializedKlibFingerprint"

 fun KlibMetadataIncrementalSerializer.serializeScope(
    ktFile: KtFile,
@@ -806,5 +818,5 @@ fun IncrementalDataProvider.getSerializedData(newSources: List<KtSourceFile>): L
 fun IncrementalDataProvider.getSerializedData(newSources: List<KtFile>): List<KotlinFileSerializedData> =
    getSerializedData(newSources.map(::KtPsiSourceFile))

-val CompilerConfiguration.incrementalDataProvider : IncrementalDataProvider?
+val CompilerConfiguration.incrementalDataProvider: IncrementalDataProvider?
    get() = get(JSConfigurationKeys.INCREMENTAL_DATA_PROVIDER)