diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt index 25a2fe4a63f..d1a552aad33 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/CacheUpdater.kt @@ -4,6 +4,7 @@ package org.jetbrains.kotlin.ir.backend.js.ic +import org.jetbrains.kotlin.backend.common.serialization.cityHash64 import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.ir.backend.js.codegen.JsGenerationGranularity @@ -63,7 +64,12 @@ class CacheUpdater( private val mainLibraryFile = KotlinLibraryFile(File(mainModule).canonicalPath) - private val cacheRootDir = File(cacheDir, "version.${compilerConfiguration.configHashForIC()}") + private val icHasher = ICHasher() + + private val cacheRootDir = run { + val configHash = icHasher.calculateConfigHash(compilerConfiguration) + File(cacheDir, "version.${configHash.hash.lowBytes.toString(Character.MAX_RADIX)}") + } fun getDirtyFileLastStats(): KotlinSourceFileMap> = dirtyFileStats @@ -77,7 +83,7 @@ class CacheUpdater( } private inner class CacheUpdaterInternal { - val signatureHashCalculator = IdSignatureHashCalculator() + val signatureHashCalculator = IdSignatureHashCalculator(icHasher) // libraries in topological order: [stdlib, ..., main] val libraryDependencies = stopwatch.measure("Resolving and loading klib dependencies") { @@ -114,7 +120,8 @@ class CacheUpdater( private val incrementalCaches = libraryDependencies.keys.associate { lib -> val libFile = KotlinLibraryFile(lib) val file = File(libFile.path) - val libraryCacheDir = File(cacheRootDir, "${file.name}.${file.absolutePath.stringHashForIC()}") + val pathHash = file.absolutePath.cityHash64().toULong().toString(Character.MAX_RADIX) + val libraryCacheDir = File(cacheRootDir, "${file.name}.$pathHash") libFile to IncrementalCache(KotlinLoadedLibraryHeader(lib), libraryCacheDir) } @@ -734,6 +741,7 @@ fun rebuildCacheForDirtyFiles( val libFile = KotlinLibraryFile(library) val dirtySrcFiles = dirtyFiles?.map { KotlinSourceFile(it) } ?: KotlinLoadedLibraryHeader(library).sourceFileFingerprints.keys + val modifiedFiles = mapOf(libFile to dirtySrcFiles.associateWith { emptyMetadata }) val jsIrLoader = JsIrLinkerLoader(configuration, dependencyGraph, emptyList(), irFactory) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/HashCalculatorForIC.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/HashCalculatorForIC.kt index 642f73aea16..c8e6bfdf806 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/HashCalculatorForIC.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/HashCalculatorForIC.kt @@ -5,42 +5,48 @@ package org.jetbrains.kotlin.ir.backend.js.ic +import org.jetbrains.kotlin.backend.common.serialization.Hash128Bits import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.config.languageVersionSettings import org.jetbrains.kotlin.ir.backend.js.transformers.irToJs.CrossModuleReferences import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrAnnotationContainer import org.jetbrains.kotlin.ir.declarations.IrFunction -import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer import org.jetbrains.kotlin.ir.symbols.IrSymbol import org.jetbrains.kotlin.ir.util.DumpIrTreeVisitor import org.jetbrains.kotlin.js.config.JSConfigurationKeys -import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.impl.buffer import org.jetbrains.kotlin.protobuf.CodedInputStream import org.jetbrains.kotlin.protobuf.CodedOutputStream -import java.io.File import java.security.MessageDigest +internal fun Hash128Bits.toProtoStream(out: CodedOutputStream) { + out.writeFixed64NoTag(lowBytes.toLong()) + out.writeFixed64NoTag(highBytes.toLong()) +} + +internal fun readHash128BitsFromProtoStream(input: CodedInputStream): Hash128Bits { + val lowBytes = input.readFixed64().toULong() + val highBytes = input.readFixed64().toULong() + return Hash128Bits(lowBytes, highBytes) +} + @JvmInline -value class ICHash(private val value: ULong = 0UL) { - fun combineWith(other: ICHash) = ICHash(value xor (other.value + 0x9e3779b97f4a7c15UL + (value shl 12) + (value shr 4))) - - override fun toString() = value.toString(16) - - fun toProtoStream(out: CodedOutputStream) = out.writeFixed64NoTag(value.toLong()) +value class ICHash(val hash: Hash128Bits = Hash128Bits()) { + fun toProtoStream(out: CodedOutputStream) = hash.toProtoStream(out) companion object { - fun fromProtoStream(input: CodedInputStream) = ICHash(input.readFixed64().toULong()) + fun fromProtoStream(input: CodedInputStream) = ICHash(readHash128BitsFromProtoStream(input)) } } private class HashCalculatorForIC { - private val md5 = MessageDigest.getInstance("MD5") + private val md5Digest = MessageDigest.getInstance("MD5") - fun update(data: ByteArray) = md5.update(data) + fun update(data: ByteArray) = md5Digest.update(data) - fun update(data: Int) = (0..3).forEach { md5.update((data shr (it * 8)).toByte()) } + fun update(data: Int) = (0..3).forEach { md5Digest.update((data shr (it * 8)).toByte()) } fun update(data: String) { update(data.length) @@ -68,87 +74,64 @@ private class HashCalculatorForIC { collection.forEach { f(it) } } - private fun bytesToULong(d: ByteArray, offset: Int): ULong { - var hash = 0UL - repeat(8) { - hash = hash or ((d[it + offset].toULong() and 0xFFUL) shl (it * 8)) - } - return hash - } - fun finalize(): ICHash { - val d = md5.digest() - return ICHash(bytesToULong(d, 0)).combineWith(ICHash(bytesToULong(d, 8))) + val hashBytes = md5Digest.digest() + md5Digest.reset() + return hashBytes.buffer.asLongBuffer().let { longBuffer -> + ICHash(Hash128Bits(longBuffer[0].toULong(), longBuffer[1].toULong())) + } } } -internal fun File.fileHashForIC(): ICHash { - val md5 = HashCalculatorForIC() - fun File.process(prefix: String = "") { - if (isDirectory) { - this.listFiles()!!.sortedBy { it.name }.forEach { - md5.update(prefix + it.name) - it.process(prefix + it.name + "/") +internal class ICHasher { + private val hashCalculator = HashCalculatorForIC() + + fun calculateConfigHash(config: CompilerConfiguration): ICHash { + val importantSettings = listOf( + JSConfigurationKeys.GENERATE_DTS, + JSConfigurationKeys.MODULE_KIND, + JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION + ) + hashCalculator.updateForEach(importantSettings) { key -> + hashCalculator.update(key.toString()) + hashCalculator.update(config.get(key).toString()) + } + + hashCalculator.update(config.languageVersionSettings.toString()) + return hashCalculator.finalize() + } + + fun calculateIrFunctionHash(function: IrFunction): ICHash { + hashCalculator.update(function) + return hashCalculator.finalize() + } + + fun calculateIrAnnotationContainerHash(container: IrAnnotationContainer): ICHash { + hashCalculator.updateAnnotationContainer(container) + return hashCalculator.finalize() + } + + fun calculateIrSymbolHash(symbol: IrSymbol): ICHash { + hashCalculator.update(symbol.toString()) + // symbol rendering prints very little information about type parameters + // TODO may be it make sense to update rendering? + (symbol.owner as? IrTypeParametersContainer)?.let { typeParameters -> + hashCalculator.updateForEach(typeParameters.typeParameters) { typeParameter -> + hashCalculator.update(typeParameter.symbol.toString()) } - } else { - md5.update(readBytes()) } + (symbol.owner as? IrFunction)?.let { irFunction -> + hashCalculator.updateForEach(irFunction.valueParameters) { functionParam -> + // symbol rendering doesn't print default params information + // it is important to understand if default params were added or removed + hashCalculator.update(functionParam.defaultValue?.let { 1 } ?: 0) + } + } + (symbol.owner as? IrAnnotationContainer)?.let(hashCalculator::updateAnnotationContainer) + return hashCalculator.finalize() } - this.process() - return md5.finalize() } -internal fun CompilerConfiguration.configHashForIC() = HashCalculatorForIC().apply { - val importantSettings = listOf( - JSConfigurationKeys.GENERATE_DTS, - JSConfigurationKeys.MODULE_KIND, - JSConfigurationKeys.PROPERTY_LAZY_INITIALIZATION - ) - updateForEach(importantSettings) { key -> - update(key.toString()) - update(get(key).toString()) - } - - update(languageVersionSettings.toString()) -}.finalize() - -internal fun IrFunction.irFunctionHashForIC() = HashCalculatorForIC().also { - it.update(this) -}.finalize() - -internal fun IrAnnotationContainer.irAnnotationContainerHashForIC() = HashCalculatorForIC().also { - it.updateAnnotationContainer(this) -}.finalize() - -internal fun IrSymbol.irSymbolHashForIC() = HashCalculatorForIC().also { - it.update(toString()) - // symbol rendering prints very little information about type parameters - // TODO may be it make sense to update rendering? - (owner as? IrTypeParametersContainer)?.let { typeParameters -> - it.updateForEach(typeParameters.typeParameters) { typeParameter -> - it.update(typeParameter.symbol.toString()) - } - } - (owner as? IrFunction)?.let { irFunction -> - it.updateForEach(irFunction.valueParameters) { functionParam -> - // symbol rendering doesn't print default params information - // it is important to understand if default params were added or removed - it.update(functionParam.defaultValue?.let { 1 } ?: 0) - } - } - (owner as? IrAnnotationContainer)?.let(it::updateAnnotationContainer) -}.finalize() - -internal fun String.stringHashForIC() = HashCalculatorForIC().also { it.update(this) }.finalize() - -internal fun KotlinLibrary.fingerprint(fileIndex: Int) = HashCalculatorForIC().apply { - update(types(fileIndex)) - update(signatures(fileIndex)) - update(strings(fileIndex)) - update(declarations(fileIndex)) - update(bodies(fileIndex)) -}.finalize() - internal fun CrossModuleReferences.crossModuleReferencesHashForIC() = HashCalculatorForIC().apply { update(moduleKind.ordinal) diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureHashCalculator.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureHashCalculator.kt index 5179b3fc3d3..7067bfd21c4 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureHashCalculator.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IdSignatureHashCalculator.kt @@ -19,7 +19,7 @@ import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptVoid import org.jetbrains.kotlin.utils.addToStdlib.ifTrue -internal class IdSignatureHashCalculator { +internal class IdSignatureHashCalculator(private val icHasher: ICHasher) { private val idSignatureSources = hashMapOf() private val idSignatureHashes = hashMapOf() @@ -29,17 +29,19 @@ internal class IdSignatureHashCalculator { private val inlineFunctionDepends = hashMapOf>() private val IrFile.annotationsHash: ICHash - get() = fileAnnotationHashes.getOrPut(this, ::irAnnotationContainerHashForIC) + get() = fileAnnotationHashes.getOrPut(this) { + icHasher.calculateIrAnnotationContainerHash(this) + } private val IrFunction.inlineFunctionFlatHash: ICHash get() = inlineFunctionFlatHashes.getOrPut(this) { val flatHash = if (isFakeOverride && this is IrSimpleFunction) { - resolveFakeOverride()?.irFunctionHashForIC() + resolveFakeOverride()?.let { icHasher.calculateIrFunctionHash(it) } ?: icError("can not resolve fake override for ${render()}") } else { - irFunctionHashForIC() + icHasher.calculateIrFunctionHash(this) } - symbol.calculateSymbolHash().combineWith(flatHash) + ICHash(symbol.calculateSymbolHash().hash.combineWith(flatHash.hash)) } private val IrFunction.inlineDepends: Collection @@ -85,7 +87,7 @@ internal class IdSignatureHashCalculator { } val fileAnnotationsHash = srcIrFile?.annotationsHash ?: ICHash() - return fileAnnotationsHash.combineWith(irSymbolHashForIC()) + return ICHash(fileAnnotationsHash.hash.combineWith(icHasher.calculateIrSymbolHash(this).hash)) } private fun IrFunction.calculateInlineFunctionTransitiveHash(): ICHash { @@ -97,7 +99,7 @@ internal class IdSignatureHashCalculator { newDependsStack.removeLast().inlineDepends.forEach { inlineFunction -> if (transitiveDepends.add(inlineFunction)) { newDependsStack += inlineFunction - transitiveHash = transitiveHash.combineWith(inlineFunction.inlineFunctionFlatHash) + transitiveHash = ICHash(transitiveHash.hash.combineWith(inlineFunction.inlineFunctionFlatHash.hash)) } } } diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt index 51020b99623..76fbb16f6cd 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IncrementalCache.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer +import org.jetbrains.kotlin.backend.common.serialization.cityHash64 +import org.jetbrains.kotlin.ir.backend.js.FingerprintHash import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.protobuf.CodedInputStream import org.jetbrains.kotlin.protobuf.CodedOutputStream @@ -40,31 +42,31 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca private class CacheHeader( val libraryFile: KotlinLibraryFile, - val libraryFingerprint: ICHash?, - val sourceFileFingerprints: Map + val libraryFingerprint: FingerprintHash?, + val sourceFileFingerprints: Map ) { constructor(library: KotlinLibraryHeader) : this(library.libraryFile, library.libraryFingerprint, library.sourceFileFingerprints) fun toProtoStream(out: CodedOutputStream) { libraryFile.toProtoStream(out) - libraryFingerprint?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile) + libraryFingerprint?.hash?.toProtoStream(out) ?: notFoundIcError("library fingerprint", libraryFile) out.writeInt32NoTag(sourceFileFingerprints.size) for ((srcFile, fingerprint) in sourceFileFingerprints) { srcFile.toProtoStream(out) - fingerprint.toProtoStream(out) + fingerprint.hash.toProtoStream(out) } } companion object { fun fromProtoStream(input: CodedInputStream): CacheHeader { val libraryFile = KotlinLibraryFile.fromProtoStream(input) - val oldLibraryFingerprint = ICHash.fromProtoStream(input) + val oldLibraryFingerprint = FingerprintHash(readHash128BitsFromProtoStream(input)) val sourceFileFingerprints = buildMapUntil(input.readInt32()) { val file = KotlinSourceFile.fromProtoStream(input) - put(file, ICHash.fromProtoStream(input)) + put(file, FingerprintHash(readHash128BitsFromProtoStream(input))) } return CacheHeader(libraryFile, oldLibraryFingerprint, sourceFileFingerprints) } @@ -76,7 +78,10 @@ internal class IncrementalCache(private val library: KotlinLibraryHeader, val ca override val directDependencies: KotlinSourceFileMap>, ) : KotlinSourceFileMetadata() - private fun KotlinSourceFile.getCacheFile(suffix: String) = File(cacheDir, "${File(path).name}.${path.stringHashForIC()}.$suffix") + private fun KotlinSourceFile.getCacheFile(suffix: String): File { + val pathHash = path.cityHash64().toULong().toString(Character.MAX_RADIX) + return File(cacheDir, "${File(path).name}.$pathHash.$suffix") + } fun buildIncrementalCacheArtifact(signatureToIndexMapping: Map>): IncrementalCacheArtifact { val klibSrcFiles = if (cacheHeaderShouldBeUpdated) { diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt index 62f36d1882f..1c4f68400bc 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/ic/KotlinLibraryHeader.kt @@ -5,12 +5,9 @@ package org.jetbrains.kotlin.ir.backend.js.ic -import org.jetbrains.kotlin.backend.common.serialization.IdSignatureDeserializer -import org.jetbrains.kotlin.backend.common.serialization.IrLibraryBytesSource -import org.jetbrains.kotlin.backend.common.serialization.IrLibraryFileFromBytes -import org.jetbrains.kotlin.backend.common.serialization.codedInputStream +import org.jetbrains.kotlin.backend.common.serialization.* import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile -import org.jetbrains.kotlin.ir.backend.js.jsOutputName +import org.jetbrains.kotlin.ir.backend.js.* import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite import java.io.File @@ -18,18 +15,26 @@ import java.io.File internal interface KotlinLibraryHeader { val libraryFile: KotlinLibraryFile - val libraryFingerprint: ICHash? + val libraryFingerprint: FingerprintHash? val sourceFileDeserializers: Map - val sourceFileFingerprints: Map + val sourceFileFingerprints: Map val jsOutputName: String? } internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : KotlinLibraryHeader { + private fun parseFingerprintsFromManifest(): Map? { + val manifestFingerprints = library.serializedIrFileFingerprints?.takeIf { it.size == sourceFiles.size } ?: return null + return sourceFiles.withIndex().associate { it.value to manifestFingerprints[it.index].fileFingerprint } + } + override val libraryFile: KotlinLibraryFile = KotlinLibraryFile(library) - override val libraryFingerprint: ICHash by lazy { File(libraryFile.path).fileHashForIC() } + override val libraryFingerprint: FingerprintHash by lazy { + val serializedKlib = library.serializedKlibFingerprint ?: SerializedKlibFingerprint(File(libraryFile.path)) + serializedKlib.klibFingerprint + } override val sourceFileDeserializers: Map by lazy { buildMapUntil(sourceFiles.size) { @@ -47,9 +52,9 @@ internal class KotlinLoadedLibraryHeader(private val library: KotlinLibrary) : K } } - override val sourceFileFingerprints: Map by lazy { - buildMapUntil(sourceFiles.size) { - put(sourceFiles[it], library.fingerprint(it)) + override val sourceFileFingerprints: Map by lazy { + parseFingerprintsFromManifest() ?: buildMapUntil(sourceFiles.size) { + put(sourceFiles[it], SerializedIrFileFingerprint(library, it).fileFingerprint) } } @@ -69,10 +74,10 @@ internal class KotlinRemovedLibraryHeader(private val libCacheDir: File) : Kotli override val libraryFile: KotlinLibraryFile get() = icError("removed library name is unavailable; cache dir: ${libCacheDir.absolutePath}") - override val libraryFingerprint: ICHash? get() = null + override val libraryFingerprint: FingerprintHash? get() = null override val sourceFileDeserializers: Map get() = emptyMap() - override val sourceFileFingerprints: Map get() = emptyMap() + override val sourceFileFingerprints: Map get() = emptyMap() override val jsOutputName: String? get() = null } diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt index 5ee64ab5155..4871f1d17dd 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/FileFingerprints.kt @@ -15,7 +15,7 @@ import java.io.File @JvmInline value class FingerprintHash(val hash: Hash128Bits) { override fun toString(): String { - return "${hash.highBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.lowBytes.toString(Character.MAX_RADIX)}" + return "${hash.lowBytes.toString(Character.MAX_RADIX)}$HASH_SEPARATOR${hash.highBytes.toString(Character.MAX_RADIX)}" } companion object { @@ -23,7 +23,7 @@ value class FingerprintHash(val hash: Hash128Bits) { internal fun fromString(s: String): FingerprintHash? { val hashes = s.split(HASH_SEPARATOR).mapNotNull { it.toULongOrNull(Character.MAX_RADIX) } - return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(it[0], it[1])) } + return hashes.takeIf { it.size == 2 }?.let { FingerprintHash(Hash128Bits(lowBytes = it[0], highBytes = it[1])) } } } } @@ -77,7 +77,7 @@ value class SerializedKlibFingerprint(val klibFingerprint: FingerprintHash) { if (isDirectory) { listFiles()!!.sortedBy { it.name }.forEach { f -> val filePrefix = "$prefix${f.name}/" - combinedHash = calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray())) + combinedHash = f.calculateKlibHash(filePrefix, cityHash128WithSeed(combinedHash, filePrefix.toByteArray())) } } else { forEachBlock { buffer, bufferSize ->