From f90db20512ba18b3492a24f73be7224a0a8bcfbd Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Thu, 22 Dec 2022 18:46:03 +0200 Subject: [PATCH] [K/N][caches] Added klib hash computing and storing --- .../ir/backend/js/ic/IncrementalCache.kt | 2 +- .../common/serialization}/FileFingerprints.kt | 36 +++++---- .../jetbrains/kotlin/ir/backend/js/klib.kt | 11 +++ .../kotlin/backend/konan/CacheStorage.kt | 5 ++ .../kotlin/backend/konan/CachedLibraries.kt | 4 + .../backend/konan/NativeGenerationState.kt | 4 + .../kotlin/backend/konan/OutputFiles.kt | 2 + .../backend/konan/lower/CacheInfoBuilder.kt | 77 +++++++++++-------- .../konan/serialization/KonanIrlinker.kt | 2 + 9 files changed, 92 insertions(+), 51 deletions(-) rename compiler/ir/{serialization.js/src/org/jetbrains/kotlin/ir/backend/js => serialization.common/src/org/jetbrains/kotlin/backend/common/serialization}/FileFingerprints.kt (83%) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index b3c15abfbed..5a2ecb2b188 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -7,7 +7,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer import org.jetbrains.kotlin.backend.common.serialization.cityHash64 -import org.jetbrains.kotlin.ir.backend.js.FingerprintHash +import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.protobuf.CodedInputStream import org.jetbrains.kotlin.protobuf.CodedOutputStream diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/FileFingerprints.kt similarity index 83% rename from compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt rename to compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/FileFingerprints.kt index c78fe107ee2..47073590d2b 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/FileFingerprints.kt @@ -3,14 +3,12 @@ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. */ -package org.jetbrains.kotlin.ir.backend.js +package org.jetbrains.kotlin.backend.common.serialization -import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits -import org.jetbrains.kotlin.backend.common.serialization.cityHash128 -import org.jetbrains.kotlin.backend.common.serialization.cityHash128WithSeed import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.library.SerializedIrFile import java.io.File +import java.nio.ByteBuffer @JvmInline value class FingerprintHash(val hash: Hash128Bits) { @@ -18,20 +16,34 @@ value class FingerprintHash(val hash: Hash128Bits) { return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}" } + fun toByteArray(): ByteArray { + val buffer = ByteBuffer.allocate(Long.SIZE_BYTES * 2) + buffer.putLong(hash.lowBytes.toLong()) + buffer.putLong(Long.SIZE_BYTES, hash.highBytes.toLong()) + return buffer.array() + } + companion object { private const val HASH_SEPARATOR = "." - internal fun fromString(s: String): FingerprintHash? { + fun fromString(s: String): FingerprintHash? { val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) } return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) } } + + fun fromByteArray(bytes: ByteArray): FingerprintHash { + val buffer = ByteBuffer.wrap(bytes) + val lowBytes = buffer.getLong(0).toULong() + val highBytes = buffer.getLong(Long.SIZE_BYTES).toULong() + return FingerprintHash(Hash128Bits(lowBytes, highBytes)) + } } } @JvmInline value class SerializedIrFileFingerprint private constructor(val fileFingerprint: FingerprintHash) { companion object { - internal fun fromString(s: String): SerializedIrFileFingerprint? { + fun fromString(s: String): SerializedIrFileFingerprint? { return FingerprintHash.fromString(s)?.let { SerializedIrFileFingerprint(it) } } @@ -92,7 +104,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) { return combinedHash } - internal fun fromString(s: String): SerializedKlibFingerprint? { + fun fromString(s: String): SerializedKlibFingerprint? { return FingerprintHash.fromString(s)?.let { SerializedKlibFingerprint(it) } } } @@ -105,13 +117,3 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) { return klibFingerprint.toString() } } - -private const val FILE_FINGERPRINTS_SEPARATOR = " " - -internal fun List.joinIrFileFingerprints(): String { - return joinToString(FILE_FINGERPRINTS_SEPARATOR) -} - -internal fun String.parseSerializedIrFileFingerprints(): List { - return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString) -} diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt index 1bfd29e3679..80013a23e3c 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt @@ -616,6 +616,17 @@ private fun getDescriptorForElement( element: PsiElement ): DeclarationDescriptor = BindingContextUtils.getNotNull(context, BindingContext.DECLARATION_TO_DESCRIPTOR, element) + +private const val FILE_FINGERPRINTS_SEPARATOR = " " + +private fun List.joinIrFileFingerprints(): String { + return joinToString(FILE_FINGERPRINTS_SEPARATOR) +} + +private fun String.parseSerializedIrFileFingerprints(): List { + return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString) +} + fun serializeModuleIntoKlib( moduleName: String, configuration: CompilerConfiguration, diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt index 51119e34570..750674a9ca1 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt @@ -25,12 +25,17 @@ internal class CacheStorage(private val generationState: NativeGenerationState) fun saveAdditionalCacheInfo() { outputFiles.prepareTempDirectories() + saveKlibContentsHash() saveCacheBitcodeDependencies() saveInlineFunctionBodies() saveClassFields() saveEagerInitializedProperties() } + private fun saveKlibContentsHash() { + outputFiles.hashFile!!.writeBytes(generationState.klibHash.toByteArray()) + } + private fun saveCacheBitcodeDependencies() { outputFiles.bitcodeDependenciesFile!!.writeLines( DependenciesSerializer.serialize(generationState.dependenciesTracker.immediateBitcodeDependencies)) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt index 9332b4545c0..67df442efe8 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt @@ -103,6 +103,9 @@ class CachedLibraries( it.absolutePath } + fun getFileHash(file: String) = + File(path).child(file).child(HASH_FILE_NAME).readBytes() + override fun computeBitcodeDependencies() = perFileBitcodeDependencies.values.flatten() override fun computeBinariesPaths() = fileDirs.map { @@ -232,6 +235,7 @@ class CachedLibraries( const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir" const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin" + const val HASH_FILE_NAME = "hash" const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps" const val INLINE_FUNCTION_BODIES_FILE_NAME = "inline_bodies" const val CLASS_FIELDS_FILE_NAME = "class_fields" diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt index 04d0533d3c1..0d17167d9dc 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.backend.konan import llvm.* +import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash +import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext import org.jetbrains.kotlin.backend.konan.driver.PhaseContext import org.jetbrains.kotlin.backend.konan.driver.utilities.BackendContextHolder @@ -64,6 +66,8 @@ internal class NativeGenerationState( ) : BasicPhaseContext(config), BackendContextHolder, LlvmIrHolder, BitcodePostProcessingContext { val outputFile = outputFiles.mainFileName + var klibHash: FingerprintHash = FingerprintHash(Hash128Bits(0U, 0U)) + val inlineFunctionBodies = mutableListOf() val classFields = mutableListOf() val eagerInitializedFiles = mutableListOf() diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt index 0ebd63565ac..8ed65a46846 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt @@ -67,6 +67,8 @@ class OutputFiles(val outputName: String, target: KonanTarget, val produce: Comp val symbolicInfoFile = "$nativeBinaryFile.dSYM" + val hashFile = tempCacheDirectory?.child(CachedLibraries.HASH_FILE_NAME) + val bitcodeDependenciesFile = tempCacheDirectory?.cacheBinaryPart()?.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME) val inlineFunctionBodiesFile = tempCacheDirectory?.cacheIrPart()?.child(CachedLibraries.INLINE_FUNCTION_BODIES_FILE_NAME) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt index 8b161b22a57..f5f8585a5b4 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt @@ -5,10 +5,13 @@ package org.jetbrains.kotlin.backend.konan.lower +import org.jetbrains.kotlin.backend.common.serialization.SerializedIrFileFingerprint +import org.jetbrains.kotlin.backend.common.serialization.SerializedKlibFingerprint import org.jetbrains.kotlin.backend.konan.DECLARATION_ORIGIN_FUNCTION_CLASS import org.jetbrains.kotlin.backend.konan.NativeGenerationState import org.jetbrains.kotlin.backend.konan.KonanFqNames import org.jetbrains.kotlin.backend.konan.isExternalObjCClass +import org.jetbrains.kotlin.backend.konan.isFunctionInterfaceFile import org.jetbrains.kotlin.backend.konan.serialization.KonanIrLinker import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr import org.jetbrains.kotlin.ir.IrElement @@ -17,54 +20,62 @@ import org.jetbrains.kotlin.ir.expressions.* import org.jetbrains.kotlin.ir.util.* import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.library.impl.javaFile internal class CacheInfoBuilder( private val generationState: NativeGenerationState, private val moduleDeserializer: KonanIrLinker.KonanPartialModuleDeserializer, private val irModule: IrModuleFragment ) { - fun build() = irModule.files.forEach { irFile -> - var hasEagerlyInitializedProperties = false + fun build() { + if (!generationState.config.producePerFileCache) + generationState.klibHash = SerializedKlibFingerprint(moduleDeserializer.klib.libraryFile.javaFile()).klibFingerprint + irModule.files.forEach { irFile -> + var hasEagerlyInitializedProperties = false - irFile.acceptChildrenVoid(object : IrElementVisitorVoid { - override fun visitElement(element: IrElement) { - element.acceptChildrenVoid(this) - } - - override fun visitClass(declaration: IrClass) { - declaration.acceptChildrenVoid(this) - - if (!declaration.isInterface && !declaration.isLocal - && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS - ) { - val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm) - generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields)) + irFile.acceptChildrenVoid(object : IrElementVisitorVoid { + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) } - } - override fun visitFunction(declaration: IrFunction) { - // Don't need to visit the children here: classes would be all local and wouldn't be handled anyway, - // as for functions - both their callees will be handled and inline bodies will be built for the top function. + override fun visitClass(declaration: IrClass) { + declaration.acceptChildrenVoid(this) - if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) { - generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration)) - trackCallees(declaration) + if (!declaration.isInterface && !declaration.isLocal + && declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS + ) { + val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm) + generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields)) + } } - } - override fun visitProperty(declaration: IrProperty) { - declaration.acceptChildrenVoid(this) + override fun visitFunction(declaration: IrFunction) { + // Don't need to visit the children here: classes would be all local and wouldn't be handled anyway, + // as for functions - both their callees will be handled and inline bodies will be built for the top function. - if (declaration.parent == irFile - && declaration.hasAnnotation(KonanFqNames.eagerInitialization) - ) { - hasEagerlyInitializedProperties = true + if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) { + generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration)) + trackCallees(declaration) + } } - } - }) - if (hasEagerlyInitializedProperties) - generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile)) + override fun visitProperty(declaration: IrProperty) { + declaration.acceptChildrenVoid(this) + + if (declaration.parent == irFile + && declaration.hasAnnotation(KonanFqNames.eagerInitialization) + ) { + hasEagerlyInitializedProperties = true + } + } + }) + + if (hasEagerlyInitializedProperties) + generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile)) + + if (generationState.config.producePerFileCache && !irFile.isFunctionInterfaceFile) + generationState.klibHash = SerializedIrFileFingerprint(moduleDeserializer.klib, moduleDeserializer.getKlibFileIndexOf(irFile)).fileFingerprint + } } private val IrDeclaration.isExported diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt index 089eb9ae596..e822fd9814f 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt @@ -501,6 +501,8 @@ internal class KonanIrLinker( ?: error("No file for $idSig") } + fun getKlibFileIndexOf(irFile: IrFile) = fileDeserializationStates.first { it.file == irFile }.fileIndex + fun buildInlineFunctionReference(irFunction: IrFunction): SerializedInlineFunctionReference { val signature = irFunction.symbol.signature ?: error("No signature for ${irFunction.render()}")