From 215da10687c8e89550a8881e5527a7b2b088ec34 Mon Sep 17 00:00:00 2001 From: Alexander Gorshenev Date: Thu, 21 Mar 2019 18:09:07 +0300 Subject: [PATCH] A little refactoring of deserializer in preparation for the next commit. --- .../serialization/IrModuleDeserializer.kt | 2 +- .../common/serialization/KotlinIrLinker.kt | 209 ++++++++++-------- .../kotlin/ir/backend/js/compiler.kt | 13 +- .../js/lower/serialization/ir/JsIrLinker.kt | 28 +-- 4 files changed, 140 insertions(+), 112 deletions(-) diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleDeserializer.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleDeserializer.kt index c80e1d9851a..6ad76fcfbbc 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleDeserializer.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/IrModuleDeserializer.kt @@ -1103,7 +1103,7 @@ abstract class IrModuleDeserializer( val originIndex = allKnownOrigins.map { it.objectInstance as IrDeclarationOriginImpl }.associateBy { it.name } fun deserializeIrDeclarationOrigin(proto: KotlinIr.IrDeclarationOrigin) = originIndex[deserializeString(proto.custom)]!! - public open fun deserializeDeclaration(proto: KotlinIr.IrDeclaration, parent: IrDeclarationParent?): IrDeclaration { + open fun deserializeDeclaration(proto: KotlinIr.IrDeclaration, parent: IrDeclarationParent?): IrDeclaration { val start = proto.coordinates.startOffset val end = proto.coordinates.endOffset diff --git a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt index 48071acea13..98cc99ea0bc 100644 --- a/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt +++ b/compiler/ir/backend.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt @@ -38,6 +38,7 @@ abstract class KotlinIrLinker( val logger: LoggingContext, val builtIns: IrBuiltIns, val symbolTable: SymbolTable, + val exportedDependencies: List, private val forwardModuleDescriptor: ModuleDescriptor?, private val firstKnownBuiltinsIndex: Long ) : DescriptorUniqIdAware, IrDeserializer { @@ -45,19 +46,24 @@ abstract class KotlinIrLinker( protected val deserializedSymbols = mutableMapOf() private val reachableTopLevels = mutableSetOf() private val deserializedTopLevels = mutableSetOf() + + //TODO: This is Native specific. Eliminate me. private val forwardDeclarations = mutableSetOf() - - private val deserializersForModules = mutableMapOf() - val resolvedForwardDeclarations = mutableMapOf() + protected val deserializersForModules = mutableMapOf() + inner class IrDeserializerForModule( private val moduleDescriptor: ModuleDescriptor, - private val moduleProto: KotlinIr.IrModule + private val moduleProto: KotlinIr.IrModule, + private val deserializationStrategy: DeserializationStrategy ) : IrModuleDeserializer(logger, builtIns, symbolTable) { private var moduleLoops = mutableMapOf() + // This is a heavy initializer + val module = deserializeIrModuleHeader(moduleProto) + private fun referenceDeserializedSymbol( proto: KotlinIr.IrSymbolData, descriptor: DeclarationDescriptor? @@ -176,12 +182,65 @@ abstract class KotlinIrLinker( isSetter = proto.isSetter, isTypeParameter = proto.isTypeParameter ) - - + // TODO: this is JS specific. Eliminate me. override fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean) = this@KotlinIrLinker.getPrimitiveTypeOrNull(symbol, hasQuestionMark) + + fun deserializeIrFile(fileProto: KotlinIr.IrFile): IrFile { + + val fileEntry = NaiveSourceBasedFileEntryImpl( + this.deserializeString(fileProto.fileEntry.name), + fileProto.fileEntry.lineStartOffsetsList.toIntArray() + ) + + // TODO: we need to store "" in protobuf, I suppose. Or better yet, reuse fqname storage from metadata. + val fqName = this.deserializeString(fileProto.fqName) + .let { if (it == "") FqName.ROOT else FqName(it) } + + val packageFragmentDescriptor = EmptyPackageFragmentDescriptor(moduleDescriptor, fqName) + + val symbol = IrFileSymbolImpl(packageFragmentDescriptor) + val file = IrFileImpl(fileEntry, symbol, fqName) + + file.annotations.addAll(deserializeAnnotations(fileProto.annotations)) + + fileProto.declarationIdList.forEach { + val uniqIdKey = it.uniqIdKey(moduleDescriptor) + reversedFileIndex.put(uniqIdKey, file) + } + + when (deserializationStrategy) { + DeserializationStrategy.EXPLICITLY_EXPORTED -> { + fileProto.explicitlyExportedToCompilerList.forEach { + val symbolProto = moduleProto.symbolTable.getSymbols(it.index) + reachableTopLevels.add(symbolProto.topLevelUniqId.uniqIdKey(moduleDescriptor)) + } + } + DeserializationStrategy.ALL -> { + fileProto.declarationIdList.forEach { + val uniqIdKey = it.uniqIdKey(moduleDescriptor) + reachableTopLevels.add(uniqIdKey) + } + } + else -> error("Unixpected deserialization strategy") + } + + return file + } + + fun deserializeIrModuleHeader( + proto: KotlinIr.IrModule + ): IrModuleFragment { + val files = proto.fileList.map { + deserializeIrFile(it) + } + val module = IrModuleFragmentImpl(moduleDescriptor, builtIns, files) + module.patchDeclarationParents(null) + return module + } } + // TODO: this is JS specific. Eliminate me. protected open fun getPrimitiveTypeOrNull(symbol: IrClassifierSymbol, hasQuestionMark: Boolean): IrSimpleType? = null protected abstract val descriptorReferenceDeserializer: DescriptorReferenceDeserializer @@ -214,10 +273,8 @@ abstract class KotlinIrLinker( private fun deserializeTopLevelDeclaration(uniqIdKey: UniqIdKey): IrDeclaration { val proto = loadTopLevelDeclarationProto(uniqIdKey) - return deserializersForModules[uniqIdKey.moduleOfOrigin]!!.deserializeDeclaration( - proto, - reversedFileIndex[uniqIdKey]!! - ) + return deserializersForModules[uniqIdKey.moduleOfOrigin]!! + .deserializeDeclaration(proto, reversedFileIndex[uniqIdKey]!!) } protected abstract fun reader(moduleDescriptor: ModuleDescriptor, uniqId: UniqId): ByteArray @@ -227,6 +284,32 @@ abstract class KotlinIrLinker( return KotlinIr.IrDeclaration.parseFrom(stream, newInstance()) } + private fun deserializeAllReachableTopLevels() { + do { + val key = reachableTopLevels.first() + val moduleOfOrigin = key.moduleOfOrigin + + if (deserializedSymbols[key]?.isBound == true || + // The key.moduleOrigin is null for uniqIds that we haven't seen in any of the library headers. + // Just skip it for now and handle it elsewhere. + moduleOfOrigin == null + ) { + + reachableTopLevels.remove(key) + deserializedTopLevels.add(key) + continue + } + + val reachable = deserializeTopLevelDeclaration(key) + val file = reversedFileIndex[key]!! + file.declarations.add(reachable) + reachable.patchDeclarationParents(file) + + reachableTopLevels.remove(key) + deserializedTopLevels.add(key) + } while (reachableTopLevels.isNotEmpty()) + } + private fun findDeserializedDeclarationForDescriptor(descriptor: DeclarationDescriptor): DeclarationDescriptor? { val topLevelDescriptor = descriptor.findTopLevelDescriptor() @@ -247,29 +330,7 @@ abstract class KotlinIrLinker( reachableTopLevels.add(topLevelKey) - do { - val key = reachableTopLevels.first() - - if (deserializedSymbols[key]?.isBound == true || - // The key.moduleOrigin is null for uniqIds that we haven't seen in any of the library headers. - // Just skip it for now and handle it elsewhere. - key.moduleOfOrigin == null - ) { - - reachableTopLevels.remove(key) - deserializedTopLevels.add(key) - continue - } - - val reachable = deserializeTopLevelDeclaration(key) - val file = reversedFileIndex[key]!! - file.declarations.add(reachable) - reachable.patchDeclarationParents(file) - - reachableTopLevels.remove(key) - deserializedTopLevels.add(key) - } while (reachableTopLevels.isNotEmpty()) - + deserializeAllReachableTopLevels() return topLevelDescriptor } @@ -294,6 +355,7 @@ abstract class KotlinIrLinker( ?: error("findDeserializedDeclaration: property descriptor $propertyDescriptor} is not present in propertyTable after deserialization}") } + // TODO: This is Native specific. Eliminate me. override fun declareForwardDeclarations() { if (forwardModuleDescriptor == null) return @@ -328,71 +390,34 @@ abstract class KotlinIrLinker( } } - fun deserializeIrFile( - fileProto: KotlinIr.IrFile, - moduleDescriptor: ModuleDescriptor, - deseralizationStrategy: DeserializationStrategy - ): IrFile { - - val moduleDeserializer = deserializersForModules[moduleDescriptor]!! - - val fileEntry = NaiveSourceBasedFileEntryImpl( - moduleDeserializer.deserializeString(fileProto.fileEntry.name), - fileProto.fileEntry.lineStartOffsetsList.toIntArray() - ) - - // TODO: we need to store "" in protobuf, I suppose. Or better yet, reuse fqname storage from metadata. - val fqName = moduleDeserializer.deserializeString(fileProto.fqName) - .let { if (it == "") FqName.ROOT else FqName(it) } - - val packageFragmentDescriptor = EmptyPackageFragmentDescriptor(moduleDescriptor, fqName) - - val symbol = IrFileSymbolImpl(packageFragmentDescriptor) - val file = IrFileImpl(fileEntry, symbol, fqName) - - fileProto.declarationIdList.forEach { - val uniqIdKey = it.uniqIdKey(moduleDescriptor) - reversedFileIndex.put(uniqIdKey, file) - - if (deseralizationStrategy == DeserializationStrategy.ALL) { - file.declarations.add(deserializeTopLevelDeclaration(uniqIdKey)) - } - } - - val annotations = moduleDeserializer.deserializeAnnotations(fileProto.annotations) - file.annotations.addAll(annotations) - - - if (deseralizationStrategy == DeserializationStrategy.EXPLICITLY_EXPORTED) - fileProto.explicitlyExportedToCompilerList.forEach { moduleDeserializer.deserializeIrSymbol(it) } - - return file - } - - fun deserializeIrModuleHeader( - proto: KotlinIr.IrModule, - moduleDescriptor: ModuleDescriptor, - deserializationStrategy: DeserializationStrategy - ): IrModuleFragment { - - deserializersForModules[moduleDescriptor] = IrDeserializerForModule(moduleDescriptor, proto) - - val files = proto.fileList.map { - deserializeIrFile(it, moduleDescriptor, deserializationStrategy) - } - val module = IrModuleFragmentImpl(moduleDescriptor, builtIns, files) - module.patchDeclarationParents(null) - return module - } - fun deserializeIrModuleHeader( moduleDescriptor: ModuleDescriptor, byteArray: ByteArray, deserializationStrategy: DeserializationStrategy = DeserializationStrategy.ONLY_REFERENCED ): IrModuleFragment { - val proto = KotlinIr.IrModule.parseFrom(byteArray.codedInputStream, newInstance()) - return deserializeIrModuleHeader(proto, moduleDescriptor, deserializationStrategy) + val deserializerForModule = deserializersForModules.getOrPut(moduleDescriptor) { + val proto = KotlinIr.IrModule.parseFrom(byteArray.codedInputStream, newInstance()) + IrDeserializerForModule(moduleDescriptor, proto, deserializationStrategy) + } + // The IrModule and its IrFiles have been created during module initialization. + return deserializerForModule.module } + + abstract val ModuleDescriptor.irHeader: ByteArray? + + fun deserializeIrModuleHeader(moduleDescriptor: ModuleDescriptor): IrModuleFragment? = + // TODO: do we really allow libraries without any IR? + moduleDescriptor.irHeader?.let { header -> + // TODO: consider skip deserializing explicitly exported declarations for libraries. + // Now it's not valid because of all dependencies that must be computed. + val deserializationStrategy = + if (exportedDependencies.contains(moduleDescriptor)) { + DeserializationStrategy.ALL + } else { + DeserializationStrategy.EXPLICITLY_EXPORTED + } + deserializeIrModuleHeader(moduleDescriptor, header, deserializationStrategy) + } } enum class DeserializationStrategy { diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt index 4faad9deb30..e569fbd70e3 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/compiler.kt @@ -9,7 +9,6 @@ import com.intellij.openapi.project.Project import org.jetbrains.kotlin.backend.common.LoggingContext import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.common.phaser.invokeToplevel -import org.jetbrains.kotlin.backend.common.serialization.DeserializationStrategy import org.jetbrains.kotlin.builtins.KotlinBuiltIns import org.jetbrains.kotlin.builtins.functions.functionInterfacePackageFragmentProvider import org.jetbrains.kotlin.config.CommonConfigurationKeys @@ -20,6 +19,8 @@ import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.backend.common.serialization.DescriptorTable +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.ir.backend.js.lower.inline.replaceUnboundSymbols import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsDeclarationTable import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer @@ -64,8 +65,9 @@ enum class CompilationMode { JS } -private val moduleHeaderFileName = "module.kji" -private val declarationsDirName = "ir/" +internal val JS_KLIBRARY_CAPABILITY = ModuleDescriptor.Capability("JS KLIBRARY") +internal val moduleHeaderFileName = "module.kji" +internal val declarationsDirName = "ir/" private val logggg = object : LoggingContext { override var inVerbosePhase: Boolean get() = TODO("not implemented") @@ -144,7 +146,7 @@ fun compile( val deserializedModuleFragments = sortedImmediateDependencies.map { val moduleFile = File(it.klibPath, moduleHeaderFileName) - deserializer.deserializeIrModuleHeader(depsDescriptors.getModuleDescriptor(it), moduleFile.readBytes(), File(it.klibPath), DeserializationStrategy.ONLY_REFERENCED) + deserializer.deserializeIrModuleHeader(depsDescriptors.getModuleDescriptor(it))!! } val moduleFragment = psi2IrTranslator.generateModuleFragment(psi2IrContext, files, deserializer) @@ -220,7 +222,8 @@ private fun loadKlibMetadata( ): ModuleDescriptorImpl { assert(isBuiltIn == (builtinsModule === null)) val builtIns = builtinsModule?.builtIns ?: object : KotlinBuiltIns(storageManager) {} - val md = ModuleDescriptorImpl(Name.special("<${moduleId.moduleName}>"), storageManager, builtIns) + val md = ModuleDescriptorImpl(Name.special("<${moduleId.moduleName}>"), storageManager, builtIns, + capabilities = mapOf(JS_KLIBRARY_CAPABILITY to File(moduleId.klibPath))) if (isBuiltIn) builtIns.builtInsModule = md val currentModuleFragmentProvider = createJsKlibMetadataPackageFragmentProvider( storageManager, md, parts.header, parts.body, metadataVersion, diff --git a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt index c95f53f1531..d3f01503681 100644 --- a/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt +++ b/compiler/ir/backend.js/src/org/jetbrains/kotlin/ir/backend/js/lower/serialization/ir/JsIrLinker.kt @@ -11,6 +11,8 @@ import org.jetbrains.kotlin.backend.common.library.DeclarationId import org.jetbrains.kotlin.backend.common.serialization.* import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.backend.js.JS_KLIBRARY_CAPABILITY +import org.jetbrains.kotlin.ir.backend.js.moduleHeaderFileName import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns import org.jetbrains.kotlin.ir.symbols.IrClassifierSymbol import org.jetbrains.kotlin.ir.util.SymbolTable @@ -20,8 +22,8 @@ class JsIrLinker( currentModule: ModuleDescriptor, logger: LoggingContext, builtIns: IrBuiltIns, - symbolTable: SymbolTable) : - KotlinIrLinker(logger, builtIns, symbolTable, null, 0x1_0000_0000L), + symbolTable: SymbolTable +) : KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null, 0x1_0000_0000L), DescriptorUniqIdAware by JsDescriptorUniqIdAware { private val FUNCTION_INDEX_START: Long = indexAfterKnownBuiltins @@ -34,20 +36,17 @@ class JsIrLinker( override val descriptorReferenceDeserializer = JsDescriptorReferenceDeserializer(currentModule, builtIns, FUNCTION_INDEX_START) - override fun reader(moduleDescriptor: ModuleDescriptor, uniqId: UniqId) = - moduleToReaderMap[moduleDescriptor]!!.declarationBytes(DeclarationId(uniqId.index, uniqId.isLocal)) - - fun deserializeIrModuleHeader( - moduleDescriptor: ModuleDescriptor, - byteArray: ByteArray, - klibLocation: File, - deserializationStrategy: DeserializationStrategy - ): IrModuleFragment { - val irFile = File(klibLocation, "ir/irCombined.knd") - moduleToReaderMap[moduleDescriptor] = CombinedIrFileReader(irFile) - return deserializeIrModuleHeader(moduleDescriptor, byteArray, deserializationStrategy) + override fun reader(moduleDescriptor: ModuleDescriptor, uniqId: UniqId): ByteArray { + val irFileReader = moduleToReaderMap.getOrPut(moduleDescriptor) { + val irFile = File(moduleDescriptor.getCapability(JS_KLIBRARY_CAPABILITY)!!, "ir/irCombined.knd") + CombinedIrFileReader(irFile) + } + return irFileReader.declarationBytes(DeclarationId(uniqId.index, uniqId.isLocal)) } + override val ModuleDescriptor.irHeader: ByteArray? get() = + this.getCapability(JS_KLIBRARY_CAPABILITY)?.let { File(it, moduleHeaderFileName).readBytes() } + override fun declareForwardDeclarations() { // since for `knownBuiltIns` such as FunctionN it is possible to have unbound symbols after deserialization // reference them through out lazy symbol table @@ -61,3 +60,4 @@ class JsIrLinker( } } } +