[K/N][caches] Added klib hash computing and storing

compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt

@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic
 
 import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
 import org.jetbrains.kotlin.backend.common.serialization.cityHash64
-import org.jetbrains.kotlin.ir.backend.js.FingerprintHash
+import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash
 import org.jetbrains.kotlin.ir.util.IdSignature
 import org.jetbrains.kotlin.protobuf.CodedInputStream
 import org.jetbrains.kotlin.protobuf.CodedOutputStream

compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/FileFingerprints.kt

@@ -3,14 +3,12 @@
  * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
  */
 
-package org.jetbrains.kotlin.ir.backend.js
+package org.jetbrains.kotlin.backend.common.serialization
 
-import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
-import org.jetbrains.kotlin.backend.common.serialization.cityHash128
-import org.jetbrains.kotlin.backend.common.serialization.cityHash128WithSeed
 import org.jetbrains.kotlin.library.KotlinLibrary
 import org.jetbrains.kotlin.library.SerializedIrFile
 import java.io.File
+import java.nio.ByteBuffer
 
 @JvmInline
 value class FingerprintHash(val hash: Hash128Bits) {
@@ -18,20 +16,34 @@ value class FingerprintHash(val hash: Hash128Bits) {
         return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}"
     }
 
+    fun toByteArray(): ByteArray {
+        val buffer = ByteBuffer.allocate(Long.SIZE_BYTES * 2)
+        buffer.putLong(hash.lowBytes.toLong())
+        buffer.putLong(Long.SIZE_BYTES, hash.highBytes.toLong())
+        return buffer.array()
+    }
+
     companion object {
         private const val HASH_SEPARATOR = "."
 
-        internal fun fromString(s: String): FingerprintHash? {
+        fun fromString(s: String): FingerprintHash? {
             val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) }
             return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) }
         }
+
+        fun fromByteArray(bytes: ByteArray): FingerprintHash {
+            val buffer = ByteBuffer.wrap(bytes)
+            val lowBytes = buffer.getLong(0).toULong()
+            val highBytes = buffer.getLong(Long.SIZE_BYTES).toULong()
+            return FingerprintHash(Hash128Bits(lowBytes, highBytes))
+        }
     }
 }
 
 @JvmInline
 value class SerializedIrFileFingerprint private constructor(val fileFingerprint: FingerprintHash) {
     companion object {
-        internal fun fromString(s: String): SerializedIrFileFingerprint? {
+        fun fromString(s: String): SerializedIrFileFingerprint? {
             return FingerprintHash.fromString(s)?.let { SerializedIrFileFingerprint(it) }
         }
 
@@ -92,7 +104,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
             return combinedHash
         }
 
-        internal fun fromString(s: String): SerializedKlibFingerprint? {
+        fun fromString(s: String): SerializedKlibFingerprint? {
             return FingerprintHash.fromString(s)?.let { SerializedKlibFingerprint(it) }
         }
     }
@@ -105,13 +117,3 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
         return klibFingerprint.toString()
     }
 }
-
-private const val FILE_FINGERPRINTS_SEPARATOR = " "
-
-internal fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
-    return joinToString(FILE_FINGERPRINTS_SEPARATOR)
-}
-
-internal fun String.parseSerializedIrFileFingerprints(): List<SerializedIrFileFingerprint> {
-    return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString)
-}

compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt

@@ -616,6 +616,17 @@ private fun getDescriptorForElement(
     element: PsiElement
 ): DeclarationDescriptor = BindingContextUtils.getNotNull(context, BindingContext.DECLARATION_TO_DESCRIPTOR, element)
 
+
+private const val FILE_FINGERPRINTS_SEPARATOR = " "
+
+private fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
+    return joinToString(FILE_FINGERPRINTS_SEPARATOR)
+}
+
+private fun String.parseSerializedIrFileFingerprints(): List<SerializedIrFileFingerprint> {
+    return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString)
+}
+
 fun serializeModuleIntoKlib(
     moduleName: String,
     configuration: CompilerConfiguration,

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt

@@ -25,12 +25,17 @@ internal class CacheStorage(private val generationState: NativeGenerationState)
 
     fun saveAdditionalCacheInfo() {
         outputFiles.prepareTempDirectories()
+        saveKlibContentsHash()
         saveCacheBitcodeDependencies()
         saveInlineFunctionBodies()
         saveClassFields()
         saveEagerInitializedProperties()
     }
 
+    private fun saveKlibContentsHash() {
+        outputFiles.hashFile!!.writeBytes(generationState.klibHash.toByteArray())
+    }
+
     private fun saveCacheBitcodeDependencies() {
         outputFiles.bitcodeDependenciesFile!!.writeLines(
                 DependenciesSerializer.serialize(generationState.dependenciesTracker.immediateBitcodeDependencies))

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt

@@ -103,6 +103,9 @@ class CachedLibraries(
                         it.absolutePath
                     }
 
+            fun getFileHash(file: String) =
+                    File(path).child(file).child(HASH_FILE_NAME).readBytes()
+
             override fun computeBitcodeDependencies() = perFileBitcodeDependencies.values.flatten()
 
             override fun computeBinariesPaths() = fileDirs.map {
@@ -232,6 +235,7 @@ class CachedLibraries(
         const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir"
         const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin"
 
+        const val HASH_FILE_NAME = "hash"
         const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps"
         const val INLINE_FUNCTION_BODIES_FILE_NAME = "inline_bodies"
         const val CLASS_FIELDS_FILE_NAME = "class_fields"

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt

@@ -6,6 +6,8 @@
 package org.jetbrains.kotlin.backend.konan
 
 import llvm.*
+import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash
+import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
 import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
 import org.jetbrains.kotlin.backend.konan.driver.utilities.BackendContextHolder
@@ -64,6 +66,8 @@ internal class NativeGenerationState(
 ) : BasicPhaseContext(config), BackendContextHolder<Context>, LlvmIrHolder, BitcodePostProcessingContext {
     val outputFile = outputFiles.mainFileName
 
+    var klibHash: FingerprintHash = FingerprintHash(Hash128Bits(0U, 0U))
+
     val inlineFunctionBodies = mutableListOf<SerializedInlineFunctionReference>()
     val classFields = mutableListOf<SerializedClassFields>()
     val eagerInitializedFiles = mutableListOf<SerializedEagerInitializedFile>()

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt

@@ -67,6 +67,8 @@ class OutputFiles(val outputName: String, target: KonanTarget, val produce: Comp
 
     val symbolicInfoFile = "$nativeBinaryFile.dSYM"
 
+    val hashFile = tempCacheDirectory?.child(CachedLibraries.HASH_FILE_NAME)
+
     val bitcodeDependenciesFile = tempCacheDirectory?.cacheBinaryPart()?.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME)
 
     val inlineFunctionBodiesFile = tempCacheDirectory?.cacheIrPart()?.child(CachedLibraries.INLINE_FUNCTION_BODIES_FILE_NAME)

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt

@@ -5,10 +5,13 @@
 
 package org.jetbrains.kotlin.backend.konan.lower
 
+import org.jetbrains.kotlin.backend.common.serialization.SerializedIrFileFingerprint
+import org.jetbrains.kotlin.backend.common.serialization.SerializedKlibFingerprint
 import org.jetbrains.kotlin.backend.konan.DECLARATION_ORIGIN_FUNCTION_CLASS
 import org.jetbrains.kotlin.backend.konan.NativeGenerationState
 import org.jetbrains.kotlin.backend.konan.KonanFqNames
 import org.jetbrains.kotlin.backend.konan.isExternalObjCClass
+import org.jetbrains.kotlin.backend.konan.isFunctionInterfaceFile
 import org.jetbrains.kotlin.backend.konan.serialization.KonanIrLinker
 import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr
 import org.jetbrains.kotlin.ir.IrElement
@@ -17,54 +20,62 @@ import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
 import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
+import org.jetbrains.kotlin.library.impl.javaFile
 
 internal class CacheInfoBuilder(
         private val generationState: NativeGenerationState,
         private val moduleDeserializer: KonanIrLinker.KonanPartialModuleDeserializer,
         private val irModule: IrModuleFragment
 ) {
-    fun build() = irModule.files.forEach { irFile ->
+    fun build() {
-        var hasEagerlyInitializedProperties = false
+        if (!generationState.config.producePerFileCache)
+            generationState.klibHash = SerializedKlibFingerprint(moduleDeserializer.klib.libraryFile.javaFile()).klibFingerprint
+        irModule.files.forEach { irFile ->
+            var hasEagerlyInitializedProperties = false
 
-        irFile.acceptChildrenVoid(object : IrElementVisitorVoid {
+            irFile.acceptChildrenVoid(object : IrElementVisitorVoid {
-            override fun visitElement(element: IrElement) {
+                override fun visitElement(element: IrElement) {
-                element.acceptChildrenVoid(this)
+                    element.acceptChildrenVoid(this)
-            }
-
-            override fun visitClass(declaration: IrClass) {
-                declaration.acceptChildrenVoid(this)
-
-                if (!declaration.isInterface && !declaration.isLocal
-                        && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS
-                ) {
-                    val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm)
-                    generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields))
                 }
-            }
 
-            override fun visitFunction(declaration: IrFunction) {
+                override fun visitClass(declaration: IrClass) {
-                // Don't need to visit the children here: classes would be all local and wouldn't be handled anyway,
+                    declaration.acceptChildrenVoid(this)
-                // as for functions - both their callees will be handled and inline bodies will be built for the top function.
 
-                if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) {
+                    if (!declaration.isInterface && !declaration.isLocal
-                    generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration))
+                            && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS
-                    trackCallees(declaration)
+                    ) {
+                        val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm)
+                        generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields))
+                    }
                 }
-            }
 
-            override fun visitProperty(declaration: IrProperty) {
+                override fun visitFunction(declaration: IrFunction) {
-                declaration.acceptChildrenVoid(this)
+                    // Don't need to visit the children here: classes would be all local and wouldn't be handled anyway,
+                    // as for functions - both their callees will be handled and inline bodies will be built for the top function.
 
-                if (declaration.parent == irFile
+                    if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) {
-                        && declaration.hasAnnotation(KonanFqNames.eagerInitialization)
+                        generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration))
-                ) {
+                        trackCallees(declaration)
-                    hasEagerlyInitializedProperties = true
+                    }
                 }
-            }
-        })
 
-        if (hasEagerlyInitializedProperties)
+                override fun visitProperty(declaration: IrProperty) {
-            generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile))
+                    declaration.acceptChildrenVoid(this)
+
+                    if (declaration.parent == irFile
+                            && declaration.hasAnnotation(KonanFqNames.eagerInitialization)
+                    ) {
+                        hasEagerlyInitializedProperties = true
+                    }
+                }
+            })
+
+            if (hasEagerlyInitializedProperties)
+                generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile))
+
+            if (generationState.config.producePerFileCache && !irFile.isFunctionInterfaceFile)
+                generationState.klibHash = SerializedIrFileFingerprint(moduleDeserializer.klib, moduleDeserializer.getKlibFileIndexOf(irFile)).fileFingerprint
+        }
     }
 
     private val IrDeclaration.isExported

kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt

@@ -501,6 +501,8 @@ internal class KonanIrLinker(
                     ?: error("No file for $idSig")
         }
 
+        fun getKlibFileIndexOf(irFile: IrFile) = fileDeserializationStates.first { it.file == irFile }.fileIndex
+
         fun buildInlineFunctionReference(irFunction: IrFunction): SerializedInlineFunctionReference {
             val signature = irFunction.symbol.signature
                     ?: error("No signature for ${irFunction.render()}")