azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[K/N][caches] Added klib hash computing and storing

Browse Source
This commit is contained in:
Igor Chevdar
2022-12-22 18:46:03 +02:00
committed by Space Team
parent 1936658e40
commit f90db20512
9 changed files with 92 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt
+1 -1
View File
@@ -7,7 +7,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer
import org.jetbrains.kotlin.backend.common.serialization.cityHash64
import org.jetbrains.kotlin.ir.backend.js.FingerprintHash
import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.protobuf.CodedInputStream
import org.jetbrains.kotlin.protobuf.CodedOutputStream
compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt → compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/FileFingerprints.kt
+19 -17
View File
@@ -3,14 +3,12 @@
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js
package org.jetbrains.kotlin.backend.common.serialization
import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
import org.jetbrains.kotlin.backend.common.serialization.cityHash128
import org.jetbrains.kotlin.backend.common.serialization.cityHash128WithSeed
import org.jetbrains.kotlin.library.KotlinLibrary
import org.jetbrains.kotlin.library.SerializedIrFile
import java.io.File
import java.nio.ByteBuffer
@JvmInline
value class FingerprintHash(val hash: Hash128Bits) {
@@ -18,20 +16,34 @@ value class FingerprintHash(val hash: Hash128Bits) {
return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}"
}
fun toByteArray(): ByteArray {
val buffer = ByteBuffer.allocate(Long.SIZE_BYTES * 2)
buffer.putLong(hash.lowBytes.toLong())
buffer.putLong(Long.SIZE_BYTES, hash.highBytes.toLong())
return buffer.array()
}
companion object {
private const val HASH_SEPARATOR = "."
internal fun fromString(s: String): FingerprintHash? {
fun fromString(s: String): FingerprintHash? {
val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) }
return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) }
}
fun fromByteArray(bytes: ByteArray): FingerprintHash {
val buffer = ByteBuffer.wrap(bytes)
val lowBytes = buffer.getLong(0).toULong()
val highBytes = buffer.getLong(Long.SIZE_BYTES).toULong()
return FingerprintHash(Hash128Bits(lowBytes, highBytes))
}
}
}
@JvmInline
value class SerializedIrFileFingerprint private constructor(val fileFingerprint: FingerprintHash) {
companion object {
internal fun fromString(s: String): SerializedIrFileFingerprint? {
fun fromString(s: String): SerializedIrFileFingerprint? {
return FingerprintHash.fromString(s)?.let { SerializedIrFileFingerprint(it) }
}
@@ -92,7 +104,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
return combinedHash
}
internal fun fromString(s: String): SerializedKlibFingerprint? {
fun fromString(s: String): SerializedKlibFingerprint? {
return FingerprintHash.fromString(s)?.let { SerializedKlibFingerprint(it) }
}
}
@@ -105,13 +117,3 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) {
return klibFingerprint.toString()
}
}
private const val FILE_FINGERPRINTS_SEPARATOR = " "
internal fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
return joinToString(FILE_FINGERPRINTS_SEPARATOR)
}
internal fun String.parseSerializedIrFileFingerprints(): List<SerializedIrFileFingerprint> {
return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString)
}
compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/klib.kt
+11
View File
@@ -616,6 +616,17 @@ private fun getDescriptorForElement(
element: PsiElement
): DeclarationDescriptor = BindingContextUtils.getNotNull(context, BindingContext.DECLARATION_TO_DESCRIPTOR, element)
private const val FILE_FINGERPRINTS_SEPARATOR = " "
private fun List<SerializedIrFileFingerprint>.joinIrFileFingerprints(): String {
return joinToString(FILE_FINGERPRINTS_SEPARATOR)
}
private fun String.parseSerializedIrFileFingerprints(): List<SerializedIrFileFingerprint> {
return split(FILE_FINGERPRINTS_SEPARATOR).mapNotNull(SerializedIrFileFingerprint::fromString)
}
fun serializeModuleIntoKlib(
moduleName: String,
configuration: CompilerConfiguration,
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CacheStorage.kt
+5
View File
@@ -25,12 +25,17 @@ internal class CacheStorage(private val generationState: NativeGenerationState)
fun saveAdditionalCacheInfo() {
outputFiles.prepareTempDirectories()
saveKlibContentsHash()
saveCacheBitcodeDependencies()
saveInlineFunctionBodies()
saveClassFields()
saveEagerInitializedProperties()
}
private fun saveKlibContentsHash() {
outputFiles.hashFile!!.writeBytes(generationState.klibHash.toByteArray())
}
private fun saveCacheBitcodeDependencies() {
outputFiles.bitcodeDependenciesFile!!.writeLines(
DependenciesSerializer.serialize(generationState.dependenciesTracker.immediateBitcodeDependencies))
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CachedLibraries.kt
+4
View File
@@ -103,6 +103,9 @@ class CachedLibraries(
it.absolutePath
}
fun getFileHash(file: String) =
File(path).child(file).child(HASH_FILE_NAME).readBytes()
override fun computeBitcodeDependencies() = perFileBitcodeDependencies.values.flatten()
override fun computeBinariesPaths() = fileDirs.map {
@@ -232,6 +235,7 @@ class CachedLibraries(
const val PER_FILE_CACHE_IR_LEVEL_DIR_NAME = "ir"
const val PER_FILE_CACHE_BINARY_LEVEL_DIR_NAME = "bin"
const val HASH_FILE_NAME = "hash"
const val BITCODE_DEPENDENCIES_FILE_NAME = "bitcode_deps"
const val INLINE_FUNCTION_BODIES_FILE_NAME = "inline_bodies"
const val CLASS_FIELDS_FILE_NAME = "class_fields"
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/NativeGenerationState.kt
+4
View File
@@ -6,6 +6,8 @@
package org.jetbrains.kotlin.backend.konan
import llvm.*
import org.jetbrains.kotlin.backend.common.serialization.FingerprintHash
import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits
import org.jetbrains.kotlin.backend.konan.driver.BasicPhaseContext
import org.jetbrains.kotlin.backend.konan.driver.PhaseContext
import org.jetbrains.kotlin.backend.konan.driver.utilities.BackendContextHolder
@@ -64,6 +66,8 @@ internal class NativeGenerationState(
) : BasicPhaseContext(config), BackendContextHolder<Context>, LlvmIrHolder, BitcodePostProcessingContext {
val outputFile = outputFiles.mainFileName
var klibHash: FingerprintHash = FingerprintHash(Hash128Bits(0U, 0U))
val inlineFunctionBodies = mutableListOf<SerializedInlineFunctionReference>()
val classFields = mutableListOf<SerializedClassFields>()
val eagerInitializedFiles = mutableListOf<SerializedEagerInitializedFile>()
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/OutputFiles.kt
+2
View File
@@ -67,6 +67,8 @@ class OutputFiles(val outputName: String, target: KonanTarget, val produce: Comp
val symbolicInfoFile = "$nativeBinaryFile.dSYM"
val hashFile = tempCacheDirectory?.child(CachedLibraries.HASH_FILE_NAME)
val bitcodeDependenciesFile = tempCacheDirectory?.cacheBinaryPart()?.child(CachedLibraries.BITCODE_DEPENDENCIES_FILE_NAME)
val inlineFunctionBodiesFile = tempCacheDirectory?.cacheIrPart()?.child(CachedLibraries.INLINE_FUNCTION_BODIES_FILE_NAME)
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/lower/CacheInfoBuilder.kt
+44 -33
View File
@@ -5,10 +5,13 @@
package org.jetbrains.kotlin.backend.konan.lower
import org.jetbrains.kotlin.backend.common.serialization.SerializedIrFileFingerprint
import org.jetbrains.kotlin.backend.common.serialization.SerializedKlibFingerprint
import org.jetbrains.kotlin.backend.konan.DECLARATION_ORIGIN_FUNCTION_CLASS
import org.jetbrains.kotlin.backend.konan.NativeGenerationState
import org.jetbrains.kotlin.backend.konan.KonanFqNames
import org.jetbrains.kotlin.backend.konan.isExternalObjCClass
import org.jetbrains.kotlin.backend.konan.isFunctionInterfaceFile
import org.jetbrains.kotlin.backend.konan.serialization.KonanIrLinker
import org.jetbrains.kotlin.backend.konan.serialization.KonanManglerIr
import org.jetbrains.kotlin.ir.IrElement
@@ -17,54 +20,62 @@ import org.jetbrains.kotlin.ir.expressions.*
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.library.impl.javaFile
internal class CacheInfoBuilder(
private val generationState: NativeGenerationState,
private val moduleDeserializer: KonanIrLinker.KonanPartialModuleDeserializer,
private val irModule: IrModuleFragment
) {
fun build() = irModule.files.forEach { irFile ->
var hasEagerlyInitializedProperties = false
fun build() {
if (!generationState.config.producePerFileCache)
generationState.klibHash = SerializedKlibFingerprint(moduleDeserializer.klib.libraryFile.javaFile()).klibFingerprint
irModule.files.forEach { irFile ->
var hasEagerlyInitializedProperties = false
irFile.acceptChildrenVoid(object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
element.acceptChildrenVoid(this)
}
override fun visitClass(declaration: IrClass) {
declaration.acceptChildrenVoid(this)
if (!declaration.isInterface && !declaration.isLocal
&& declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS
) {
val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm)
generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields))
irFile.acceptChildrenVoid(object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
element.acceptChildrenVoid(this)
}
}
override fun visitFunction(declaration: IrFunction) {
// Don't need to visit the children here: classes would be all local and wouldn't be handled anyway,
// as for functions - both their callees will be handled and inline bodies will be built for the top function.
override fun visitClass(declaration: IrClass) {
declaration.acceptChildrenVoid(this)
if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) {
generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration))
trackCallees(declaration)
if (!declaration.isInterface && !declaration.isLocal
&& declaration.isExported && declaration.origin != DECLARATION_ORIGIN_FUNCTION_CLASS
) {
val declaredFields = generationState.context.getLayoutBuilder(declaration).getDeclaredFields(generationState.llvm)
generationState.classFields.add(moduleDeserializer.buildClassFields(declaration, declaredFields))
}
}
}
override fun visitProperty(declaration: IrProperty) {
declaration.acceptChildrenVoid(this)
override fun visitFunction(declaration: IrFunction) {
// Don't need to visit the children here: classes would be all local and wouldn't be handled anyway,
// as for functions - both their callees will be handled and inline bodies will be built for the top function.
if (declaration.parent == irFile
&& declaration.hasAnnotation(KonanFqNames.eagerInitialization)
) {
hasEagerlyInitializedProperties = true
if (!declaration.isFakeOverride && declaration.isInline && declaration.isExported) {
generationState.inlineFunctionBodies.add(moduleDeserializer.buildInlineFunctionReference(declaration))
trackCallees(declaration)
}
}
}
})
if (hasEagerlyInitializedProperties)
generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile))
override fun visitProperty(declaration: IrProperty) {
declaration.acceptChildrenVoid(this)
if (declaration.parent == irFile
&& declaration.hasAnnotation(KonanFqNames.eagerInitialization)
) {
hasEagerlyInitializedProperties = true
}
}
})
if (hasEagerlyInitializedProperties)
generationState.eagerInitializedFiles.add(moduleDeserializer.buildEagerInitializedFile(irFile))
if (generationState.config.producePerFileCache && !irFile.isFunctionInterfaceFile)
generationState.klibHash = SerializedIrFileFingerprint(moduleDeserializer.klib, moduleDeserializer.getKlibFileIndexOf(irFile)).fileFingerprint
}
}
private val IrDeclaration.isExported
kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt
+2
View File
@@ -501,6 +501,8 @@ internal class KonanIrLinker(
?: error("No file for $idSig")
}
fun getKlibFileIndexOf(irFile: IrFile) = fileDeserializationStates.first { it.file == irFile }.fileIndex
fun buildInlineFunctionReference(irFunction: IrFunction): SerializedInlineFunctionReference {
val signature = irFunction.symbol.signature
?: error("No signature for ${irFunction.render()}")