JS IC: IC lowerings prototype

This commit is contained in:
Anton Bannykh
2021-07-06 17:38:09 +03:00
committed by teamcityserver
parent 525c5b886f
commit 20088994c1
22 changed files with 1830 additions and 62 deletions
@@ -8,6 +8,8 @@ dependencies {
api(project(":compiler:ir.serialization.common"))
api(project(":js:js.frontend"))
implementation(project(":compiler:ir.backend.common"))
compile(project(":compiler:ir.tree.persistent"))
compileOnly(project(":kotlin-reflect-api"))
compileOnly(intellijCoreDep()) { includeJars("intellij-core") }
}
@@ -0,0 +1,202 @@
/*
* Copyright 2010-2020 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js
import org.jetbrains.kotlin.backend.common.DefaultMapping
import org.jetbrains.kotlin.backend.common.DelegateFactory
import org.jetbrains.kotlin.backend.common.Mapping
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.library.impl.*
fun JsMapping(irFactory: IrFactory) = JsMapping(JsMappingState(irFactory))
class JsMapping(val state: JsMappingState) : DefaultMapping(state) {
val outerThisFieldSymbols = state.newDeclarationToDeclarationMapping<IrClass, IrField>()
val innerClassConstructors = state.newDeclarationToDeclarationMapping<IrConstructor, IrConstructor>()
val originalInnerClassPrimaryConstructorByClass = state.newDeclarationToDeclarationMapping<IrClass, IrConstructor>()
val secondaryConstructorToDelegate = state.newDeclarationToDeclarationMapping<IrConstructor, IrSimpleFunction>()
val secondaryConstructorToFactory = state.newDeclarationToDeclarationMapping<IrConstructor, IrSimpleFunction>()
val objectToGetInstanceFunction = state.newDeclarationToDeclarationMapping<IrClass, IrSimpleFunction>()
val objectToInstanceField = state.newDeclarationToDeclarationMapping<IrClass, IrField>()
val classToSyntheticPrimaryConstructor = state.newDeclarationToDeclarationMapping<IrClass, IrConstructor>()
val privateMemberToCorrespondingStatic = state.newDeclarationToDeclarationMapping<IrFunction, IrSimpleFunction>()
val constructorToInitFunction = state.newDeclarationToDeclarationMapping<IrConstructor, IrSimpleFunction>()
val enumEntryToGetInstanceFun = state.newDeclarationToDeclarationMapping<IrEnumEntry, IrSimpleFunction>()
val enumEntryToInstanceField = state.newDeclarationToDeclarationMapping<IrEnumEntry, IrField>()
val enumConstructorToNewConstructor = state.newDeclarationToDeclarationMapping<IrConstructor, IrConstructor>()
val enumClassToCorrespondingEnumEntry = state.newDeclarationToDeclarationMapping<IrClass, IrEnumEntry>()
val enumConstructorOldToNewValueParameters = state.newDeclarationToDeclarationMapping<IrValueDeclaration, IrValueParameter>()
val enumEntryToCorrespondingField = state.newDeclarationToDeclarationMapping<IrEnumEntry, IrField>()
val enumClassToInitEntryInstancesFun = state.newDeclarationToDeclarationMapping<IrClass, IrSimpleFunction>()
}
class JsMappingState(val irFactory: IrFactory) : DelegateFactory {
override fun <K : IrDeclaration, V : IrDeclaration> newDeclarationToDeclarationMapping(): Mapping.Delegate<K, V> {
return JsMappingDelegate<K, V>(irFactory).also {
allMappings += it
}
}
override fun <K : IrDeclaration, V : Collection<IrDeclaration>> newDeclarationToDeclarationCollectionMapping(): Mapping.Delegate<K, V> {
return JsMappingCollectionDelegate<K, V>(irFactory).also {
allMappings += it
}
}
private val allMappings = mutableListOf<SerializableMapping>()
fun serializeMappings(declarations: Iterable<IrDeclaration>, symbolSerializer: (IrSymbol) -> Long): SerializedMappings {
return SerializedMappings(allMappings.map { mapping ->
val keys = mutableListOf<Long>()
val values = mutableListOf<ByteArray>()
declarations.forEach { d ->
mapping.serializeMapping(d, symbolSerializer)?.let { bytes ->
keys += symbolSerializer((d as IrSymbolOwner).symbol)
values += bytes
}
}
SerializedMapping(
IrMemoryLongArrayWriter(keys).writeIntoMemory(),
IrMemoryArrayWriter(values).writeIntoMemory(),
)
})
}
fun mappingsDeserializer(mapping: SerializedMappings, signatureDeserializer: (Long) -> IdSignature, symbolDeserializer: (Long) -> IrSymbol): (IdSignature, IrDeclaration) -> Unit {
if (allMappings.size != mapping.mappings.size) error("Mapping size mismatch")
val index = Array<Map<IdSignature, ByteArray>>(allMappings.size) { i ->
val bytes = mapping.mappings[i]
val s = IrLongArrayMemoryReader(bytes.keys).array.map(signatureDeserializer)
val v = IrArrayMemoryReader(bytes.values).toArray()
if (s.size != v.size) error("Keys size != values size")
s.withIndex().associate { it.value to v[it.index] }
}
return { signature, declaration ->
for (i in allMappings.indices) {
index[i][signature]?.let { bytes ->
allMappings[i].loadMapping(declaration, bytes, symbolDeserializer)
}
}
}
}
}
class SerializedMappings(
val mappings: List<SerializedMapping>
)
class SerializedMapping(
val keys: ByteArray,
val values: ByteArray,
)
private interface SerializableMapping {
fun serializeMapping(declaration: IrDeclaration, symbolSerializer: (IrSymbol) -> Long): ByteArray?
fun loadMapping(declaration: IrDeclaration, mapping: ByteArray, symbolDeserializer: (Long) -> IrSymbol)
}
private class JsMappingDelegate<K : IrDeclaration, V : IrDeclaration>(val irFactory: IrFactory) : Mapping.Delegate<K, V>(), SerializableMapping {
private val map: MutableMap<IrSymbol, IrSymbol> = mutableMapOf()
@Suppress("UNCHECKED_CAST")
override operator fun get(key: K): V? {
irFactory.stageController.lazyLower(key)
return map[(key as IrSymbolOwner).symbol]?.owner as? V
}
override operator fun set(key: K, value: V?) {
irFactory.stageController.lazyLower(key)
if (value == null) {
map.remove((key as IrSymbolOwner).symbol)
} else {
map[(key as IrSymbolOwner).symbol] = (value as IrSymbolOwner).symbol
}
}
override fun serializeMapping(declaration: IrDeclaration, symbolSerializer: (IrSymbol) -> Long): ByteArray? {
return map[(declaration as IrSymbolOwner).symbol]?.let { symbol ->
symbolSerializer(symbol).toByteArray()
}
}
override fun loadMapping(declaration: IrDeclaration, mapping: ByteArray, symbolDeserializer: (Long) -> IrSymbol) {
map[(declaration as IrSymbolOwner).symbol] = symbolDeserializer(mapping.toLong())
}
override val keys: Set<K>
get() = TODO("Not yet implemented")
override val values: Collection<V>
get() = TODO("Not yet implemented")
}
private class JsMappingCollectionDelegate<K : IrDeclaration, V : Collection<IrDeclaration>>(val irFactory: IrFactory) : Mapping.Delegate<K, V>(), SerializableMapping {
private val map: MutableMap<IrSymbol, Collection<IrSymbol>> = mutableMapOf()
@Suppress("UNCHECKED_CAST")
override operator fun get(key: K): V? {
irFactory.stageController.lazyLower(key)
return map[(key as IrSymbolOwner).symbol]?.map { it.owner as IrDeclaration } as? V
}
override operator fun set(key: K, value: V?) {
irFactory.stageController.lazyLower(key)
if (value == null) {
map.remove((key as IrSymbolOwner).symbol)
} else {
map[(key as IrSymbolOwner).symbol] = value.map { (it as IrSymbolOwner).symbol }
}
}
override fun serializeMapping(declaration: IrDeclaration, symbolSerializer: (IrSymbol) -> Long): ByteArray? {
return map[(declaration as IrSymbolOwner).symbol]?.let { symbols ->
IrMemoryLongArrayWriter(symbols.map(symbolSerializer)).writeIntoMemory()
}
}
override fun loadMapping(declaration: IrDeclaration, mapping: ByteArray, symbolDeserializer: (Long) -> IrSymbol) {
map[(declaration as IrSymbolOwner).symbol] = IrLongArrayMemoryReader(mapping).array.map(symbolDeserializer)
}
override val keys: Set<K>
get() = TODO("Not yet implemented")
override val values: Collection<V>
get() = TODO("Not yet implemented")
}
fun ByteArray.toLong(): Long {
var result = this[0].toLong() and 0xFFL
for (i in 1..7) {
result = (result shl 8) or (this[i].toLong() and 0xFFL)
}
return result
}
fun Long.toByteArray(): ByteArray {
val result = ByteArray(8)
var self = this
for (i in 7 downTo 0) {
result[i] = self.toByte()
self = self ushr 8
}
return result
}
@@ -0,0 +1,313 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.overrides.DefaultFakeOverrideClassFilter
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.backend.js.JsMappingState
import org.jetbrains.kotlin.ir.backend.js.JsStatementOrigins
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.declarations.IrDeclaration
import org.jetbrains.kotlin.ir.declarations.IrFile
import org.jetbrains.kotlin.ir.expressions.IrStatementOriginImpl
import org.jetbrains.kotlin.ir.serialization.CarrierDeserializer
import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.library.SerializedIrFile
import org.jetbrains.kotlin.library.impl.DeclarationId
import org.jetbrains.kotlin.library.impl.DeclarationIrTableMemoryReader
import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader
import org.jetbrains.kotlin.library.impl.IrLongArrayMemoryReader
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoIrFile
class IcFileDeserializer(
val linker: JsIrLinker,
file: IrFile,
originalFileReader: IrLibraryFile,
fileProto: org.jetbrains.kotlin.backend.common.serialization.proto.IrFile,
deserializeBodies: Boolean,
allowErrorNodes: Boolean,
deserializeInlineFunctions: Boolean,
val moduleDeserializer: IrModuleDeserializer,
useGlobalSignatures: Boolean,
val handleNoModuleDeserializerFound: (IdSignature, ModuleDescriptor, Collection<IrModuleDeserializer>) -> IrModuleDeserializer,
val originalEnqueue: IdSignature.(IcFileDeserializer) -> Unit,
val icFileData: SerializedIcDataForFile,
val mappingState: JsMappingState,
val publicSignatureToIcFileDeserializer: MutableMap<IdSignature, IcFileDeserializer>,
val enqueue: IdSignature.(IcFileDeserializer) -> Unit,
) {
val originalSymbolDeserializer =
IrSymbolDeserializer(
linker.symbolTable,
originalFileReader,
file.symbol,
fileProto.actualList,
{ idSig ->
idSig.enqueue(this)
if (idSig.hasTopLevel) {
idSig.topLevelSignature().originalEnqueue(this)
}
},
linker::handleExpectActualMapping,
useGlobalSignatures = useGlobalSignatures,
enqueueAllDeclarations = true,
deserializePublicSymbol = ::deserializeOriginalPublicSymbol,
)
private val originalDeclarationDeserializer = IrDeclarationDeserializer(
linker.builtIns,
linker.symbolTable,
linker.symbolTable.irFactory,
originalFileReader,
file,
allowErrorNodes,
deserializeInlineFunctions,
deserializeBodies,
originalSymbolDeserializer,
linker.fakeOverrideBuilder.platformSpecificClassFilter,
linker.fakeOverrideBuilder,
allowRedeclaration = true,
)
private fun deserializeOriginalPublicSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol {
assert(idSig.isPublic)
val topLevelSig = idSig.topLevelSignature()
if (idSig in originalFileDeserializer.reversedSignatureIndex) {
topLevelSig.originalEnqueue(this)
idSig.enqueue(this)
linker.modulesWithReachableTopLevels.add(moduleDeserializer)
return originalFileDeserializer.symbolDeserializer.deserializeIrSymbol(idSig, symbolKind).also {
linker.deserializedSymbols.add(it)
}
} else {
val actualModuleDeserializer =
moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) ?: handleNoModuleDeserializerFound(
idSig,
moduleDeserializer.moduleDescriptor,
moduleDeserializer.moduleDependencies
)
return actualModuleDeserializer.deserializeIrSymbol(idSig, symbolKind)
}
}
val originalFileDeserializer = IrFileDeserializer(file, originalFileReader, fileProto, originalSymbolDeserializer, originalDeclarationDeserializer)
val originalVisited = HashSet<IdSignature>()
val originalSignatureQueue = ArrayDeque<IdSignature>() // Top-level signatures to be deserialized from original KLIB
// Returns whether this file should be queued for deserialization
fun enqueueForDeserialization(idSig: IdSignature): Boolean {
if (originalVisited.add(idSig)) {
originalSignatureQueue.addLast(idSig)
return originalSignatureQueue.size == 1
}
return false
}
fun deserializePendingSignatures() {
while (!originalSignatureQueue.isEmpty()) {
val signature = originalSignatureQueue.removeFirst()
deserializeAnyDeclaration(signature)
}
}
// Explicitly exported declarations (e.g. top-level initializers) must be deserialized before all other declarations.
// Thus we schedule their deserialization in deserializer's constructor.
val explicitlyExportedToCompiler: Collection<IdSignature> = fileProto.explicitlyExportedToCompilerList.map {
val symbolData = originalSymbolDeserializer.parseSymbolData(it)
originalSymbolDeserializer.deserializeIdSignature(symbolData.signatureId)
}
fun allOriginalDeclarationSignatures(): Collection<IdSignature> = originalFileDeserializer.reversedSignatureIndex.keys
// IC data processing starts here
private val icFileReader = FileReaderFromSerializedIrFile(icFileData.file)
val symbolDeserializer = IrSymbolDeserializer(
linker.symbolTable,
icFileReader,
file.symbol,
emptyList(),
{ idSig -> enqueueLocalTopLevelDeclaration(idSig) },
{ _, s -> s },
enqueueAllDeclarations = true,
useGlobalSignatures = true,
deserializedSymbols = originalFileDeserializer.symbolDeserializer.deserializedSymbols,
::deserializePublicSymbol
)
private val declarationDeserializer = IrDeclarationDeserializer(
linker.builtIns,
linker.symbolTable,
linker.symbolTable.irFactory,
icFileReader,
file,
allowErrorNodes = true,
deserializeInlineFunctions = true,
deserializeBodies = true,
symbolDeserializer,
DefaultFakeOverrideClassFilter,
linker.fakeOverrideBuilder,
skipMutableState = true,
additionalStatementOriginIndex = additionalStatementOriginIndex,
allowErrorStatementOrigins = true,
allowRedeclaration = true,
)
private val protoFile: ProtoIrFile by lazy { ProtoIrFile.parseFrom(icFileData.file.fileData.codedInputStream, ExtensionRegistryLite.newInstance()) }
private val carrierDeserializer by lazy { CarrierDeserializer(declarationDeserializer, icFileData.carriers) }
val reversedSignatureIndex: Map<IdSignature, Int> by lazy { protoFile.declarationIdList.map { symbolDeserializer.deserializeIdSignature(it) to it }.toMap() }
val visited = HashSet<IdSignature>()
val mappingsDeserializer by lazy {
mappingState.mappingsDeserializer(icFileData.mappings, { code ->
val symbolData = symbolDeserializer.parseSymbolData(code)
symbolDeserializer.deserializeIdSignature(symbolData.signatureId)
}) {
deserializeIrSymbol(it)
}
}
fun init() {
reversedSignatureIndex.keys.forEach {
publicSignatureToIcFileDeserializer[it] = this
}
}
private val containerSigToOrder by lazy {
mutableMapOf<IdSignature, ByteArray>().also { map ->
val containerIds = IrLongArrayMemoryReader(icFileData.order.containerSignatures).array
val declarationIds = IrArrayMemoryReader(icFileData.order.declarationSignatures)
containerIds.forEachIndexed { index, id ->
val symbolData = symbolDeserializer.parseSymbolData(id)
val containerSig = symbolDeserializer.deserializeIdSignature(symbolData.signatureId)
map[containerSig] = declarationIds.tableItemBytes(index)
}
}
}
fun loadClassOrder(classSignature: IdSignature): List<IrSymbol>? {
val bytes = containerSigToOrder[classSignature] ?: return null
return IrLongArrayMemoryReader(bytes).array.map(::deserializeIrSymbol)
}
private fun deserializePublicSymbol(idSig: IdSignature, kind: BinarySymbolData.SymbolKind) : IrSymbol {
// TODO: reference lowered declarations cross-module
val topLevelSig = idSig.topLevelSignature()
val actualModuleDeserializer =
moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) ?:
handleNoModuleDeserializerFound(
idSig,
moduleDeserializer.moduleDescriptor,
moduleDeserializer.moduleDependencies
)
return actualModuleDeserializer.deserializeIrSymbol(idSig, kind)
}
private fun enqueueLocalTopLevelDeclaration(idSig: IdSignature) {
// We only care about declarations from IC cache. They all are in the map.
val deser = publicSignatureToIcFileDeserializer[idSig] ?: return
idSig.enqueue(deser)
}
fun deserializeDeclaration(idSig: IdSignature): IrDeclaration? {
cachedDeclaration(idSig)?.let { return it }
val idSigIndex = reversedSignatureIndex[idSig] ?: return null
val declarationStream = icFileReader.irDeclaration(idSigIndex).codedInputStream
val declarationProto = org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration.parseFrom(declarationStream, ExtensionRegistryLite.newInstance())
return declarationDeserializer.deserializeDeclaration(declarationProto)
}
// Return declaration iff it was already deserialized
private fun cachedDeclaration(idSig: IdSignature): IrDeclaration? {
val symbol = symbolDeserializer.deserializedSymbols[idSig] // Same map is used for both symbol deserializers
if (symbol != null && symbol.isBound) return symbol.owner as? IrDeclaration
return null
}
fun deserializeAnyDeclaration(idSig: IdSignature): IrDeclaration? {
if (idSig is IdSignature.FileSignature) return null // TODO: is it needed
cachedDeclaration(idSig)?.let { return it }
// TODO fast path?
val maybeTopLevel = if (!idSig.isLocal || idSig.hasTopLevel) idSig.topLevelSignature() else idSig
if (maybeTopLevel in originalFileDeserializer.reversedSignatureIndex.keys) {
originalFileDeserializer.deserializeFileImplicitDataIfFirstUse()
originalFileDeserializer.deserializeDeclaration(maybeTopLevel)
// At this point the declaration should've been deserialized
return cachedDeclaration(idSig) // Will be null in case of fake overrides
} else if (maybeTopLevel in reversedSignatureIndex) {
return deserializeDeclaration(maybeTopLevel)
}
// TODO: error?
return null
}
fun deserializeIrSymbol(code: Long): IrSymbol {
return symbolDeserializer.deserializeIrSymbol(code)
}
fun deserializeIrSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol {
idSig.enqueue(this)
return symbolDeserializer.deserializeIrSymbol(idSig, symbolKind)
}
fun injectCarriers(declaration: IrDeclaration, signature: IdSignature) {
carrierDeserializer.injectCarriers(declaration, signature)
}
companion object {
private val additionalStatementOrigins = JsStatementOrigins::class.nestedClasses.toList()
private val additionalStatementOriginIndex =
additionalStatementOrigins.mapNotNull { it.objectInstance as? IrStatementOriginImpl }.associateBy { it.debugName }
}
}
private class FileReaderFromSerializedIrFile(val irFile: SerializedIrFile) : IrLibraryFile() {
private val declarationReader = DeclarationIrTableMemoryReader(irFile.declarations)
private val typeReader = IrArrayMemoryReader(irFile.types)
private val signatureReader = IrArrayMemoryReader(irFile.signatures)
private val stringReader = IrArrayMemoryReader(irFile.strings)
private val bodyReader = IrArrayMemoryReader(irFile.bodies)
override fun irDeclaration(index: Int): ByteArray = declarationReader.tableItemBytes(DeclarationId(index))
override fun type(index: Int): ByteArray = typeReader.tableItemBytes(index)
override fun signature(index: Int): ByteArray = signatureReader.tableItemBytes(index)
override fun string(index: Int): ByteArray = stringReader.tableItemBytes(index)
override fun body(index: Int): ByteArray = bodyReader.tableItemBytes(index)
}
@@ -0,0 +1,271 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.backend.js.JsMapping
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.declarations.impl.IrModuleFragmentImpl
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory
import org.jetbrains.kotlin.ir.symbols.IrFileSymbol
import org.jetbrains.kotlin.ir.symbols.IrPropertySymbol
import org.jetbrains.kotlin.ir.symbols.IrSimpleFunctionSymbol
import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.ir.util.isEffectivelyExternal
import org.jetbrains.kotlin.library.IrLibrary
import org.jetbrains.kotlin.library.impl.IrLongArrayMemoryReader
import org.jetbrains.kotlin.protobuf.ExtensionRegistryLite
import org.jetbrains.kotlin.backend.common.serialization.proto.IrFile as ProtoFile
class IcModuleDeserializer(
val irFactory: PersistentIrFactory,
val mapping: JsMapping,
val linker: JsIrLinker,
val icData: SerializedIcData,
moduleDescriptor: ModuleDescriptor,
override val klib: IrLibrary,
override val strategy: DeserializationStrategy,
private val containsErrorCode: Boolean = false,
private val useGlobalSignatures: Boolean = false,
) : IrModuleDeserializer(moduleDescriptor) {
private val fileToDeserializerMap = mutableMapOf<IrFile, IrFileDeserializer>()
internal val moduleReversedFileIndex = mutableMapOf<IdSignature, IcFileDeserializer>()
internal val icModuleReversedFileIndex = mutableMapOf<IdSignature, IcFileDeserializer>()
override val moduleDependencies by lazy {
moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.map { linker.resolveModuleDeserializer(it, null) }
}
override fun fileDeserializers(): Collection<IrFileDeserializer> {
return fileToDeserializerMap.values
}
override fun init(delegate: IrModuleDeserializer) {
val fileCount = klib.fileCount()
val files = ArrayList<IrFile>(fileCount)
for (i in 0 until fileCount) {
val fileStream = klib.file(i).codedInputStream
val fileProto = ProtoFile.parseFrom(fileStream, ExtensionRegistryLite.newInstance())
files.add(deserializeIrFile(fileProto, i, delegate, containsErrorCode))
}
moduleFragment.files.addAll(files)
fileToDeserializerMap.values.forEach { it.symbolDeserializer.deserializeExpectActualMapping() }
}
private fun IrSymbolDeserializer.deserializeExpectActualMapping() {
actuals.forEach {
val expectSymbol = parseSymbolData(it.expectSymbol)
val actualSymbol = parseSymbolData(it.actualSymbol)
val expect = deserializeIdSignature(expectSymbol.signatureId)
val actual = deserializeIdSignature(actualSymbol.signatureId)
assert(linker.expectUniqIdToActualUniqId[expect] == null) {
"Expect signature $expect is already actualized by ${linker.expectUniqIdToActualUniqId[expect]}, while we try to record $actual"
}
linker.expectUniqIdToActualUniqId[expect] = actual
// Non-null only for topLevel declarations.
findModuleDeserializerForTopLevelId(actual)?.let { md -> linker.topLevelActualUniqItToDeserializer[actual] = md }
}
}
override fun referenceSimpleFunctionByLocalSignature(file: IrFile, idSignature: IdSignature): IrSimpleFunctionSymbol =
fileToDeserializerMap[file]?.symbolDeserializer?.referenceSimpleFunctionByLocalSignature(idSignature)
?: error("No deserializer for file $file in module ${moduleDescriptor.name}")
override fun referencePropertyByLocalSignature(file: IrFile, idSignature: IdSignature): IrPropertySymbol =
fileToDeserializerMap[file]?.symbolDeserializer?.referencePropertyByLocalSignature(idSignature)
?: error("No deserializer for file $file in module ${moduleDescriptor.name}")
// TODO: fix to topLevel checker
override fun contains(idSig: IdSignature): Boolean = idSig in moduleReversedFileIndex || idSig in icModuleReversedFileIndex
override fun deserializeIrSymbol(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol {
assert(idSig.isPublic)
if (idSig in icModuleReversedFileIndex) {
val icDeserializer = icModuleReversedFileIndex[idSig]!!
return icDeserializer.deserializeIrSymbol(idSig, symbolKind)
}
val topLevelSignature = idSig.topLevelSignature()
val icDeserializer = moduleReversedFileIndex[topLevelSignature]
?: error("No file for $topLevelSignature (@ $idSig) in module $moduleDescriptor")
topLevelSignature.originalEnqueue(icDeserializer)
idSig.enqueue(icDeserializer)
linker.modulesWithReachableTopLevels.add(this)
return icDeserializer.originalFileDeserializer.symbolDeserializer.deserializeIrSymbol(idSig, symbolKind).also {
linker.deserializedSymbols.add(it)
}
}
override val moduleFragment: IrModuleFragment = IrModuleFragmentImpl(moduleDescriptor, linker.builtIns, emptyList())
private val pathToIcFileData = icData.files.associateBy {
it.file.path
}
private val publicSignatureToIcFileDeserializer = mutableMapOf<IdSignature, IcFileDeserializer>()
private fun deserializeIrFile(
fileProto: ProtoFile,
fileIndex: Int,
moduleDeserializer: IrModuleDeserializer,
allowErrorNodes: Boolean
): IrFile {
val fileReader = IrLibraryFileFromKlib(moduleDeserializer.klib, fileIndex)
val file = fileReader.createFile(moduleFragment, fileProto)
val icFileData = pathToIcFileData[file.path]!!
val icDeserializer = IcFileDeserializer(
linker,
file,
fileReader,
fileProto,
strategy.needBodies,
allowErrorNodes,
strategy.inlineBodies,
moduleDeserializer,
useGlobalSignatures,
linker::handleNoModuleDeserializerFound,
{ fileDeserializer -> originalEnqueue(fileDeserializer) },
icFileData,
mapping.state,
publicSignatureToIcFileDeserializer,
{ fileDeserializer -> enqueue(fileDeserializer) },
)
icDeserializers += icDeserializer
icDeserializer.explicitlyExportedToCompiler.forEach { it.topLevelSignature().originalEnqueue(icDeserializer) }
fileToDeserializerMap[file] = icDeserializer.originalFileDeserializer
val topLevelDeclarations = icDeserializer.originalFileDeserializer.reversedSignatureIndex.keys
topLevelDeclarations.forEach {
moduleReversedFileIndex.putIfAbsent(it, icDeserializer) // TODO Why not simple put?
}
if (strategy.theWholeWorld) {
icDeserializer.allOriginalDeclarationSignatures().forEach { it.originalEnqueue(icDeserializer) }
}
if (strategy.theWholeWorld || strategy.explicitlyExported) {
linker.modulesWithReachableTopLevels.add(this)
}
return file
}
override fun addModuleReachableTopLevel(idSig: IdSignature) {
val fileLocalDeserializationState = moduleReversedFileIndex[idSig] ?: error("No file found for key $idSig")
idSig.originalEnqueue(fileLocalDeserializationState)
}
override fun deserializeReachableDeclarations() {
while (!originalFileQueue.isEmpty()) {
originalFileQueue.removeFirst().deserializePendingSignatures()
}
}
val originalFileQueue = ArrayDeque<IcFileDeserializer>()
fun IdSignature.originalEnqueue(fileDeserializer: IcFileDeserializer) {
if (fileDeserializer.enqueueForDeserialization(this)) {
originalFileQueue.addLast(fileDeserializer)
}
}
val fileQueue = ArrayDeque<IcFileDeserializer>()
val signatureQueue = ArrayDeque<IdSignature>()
val icDeserializers = mutableListOf<IcFileDeserializer>()
val classToDeclarationSymbols = mutableMapOf<IrClass, List<IrSymbol>>()
fun IdSignature.enqueue(icDeserializer: IcFileDeserializer) {
if (this !in icDeserializer.visited) {
fileQueue.addLast(icDeserializer)
signatureQueue.addLast(this)
icDeserializer.visited += this
}
}
override fun postProcess() {
icDeserializers.forEach { icDeserializer ->
if (!icDeserializer.visited.isEmpty()) {
val file = icDeserializer.originalFileDeserializer.file
icDeserializer.init()
icDeserializer.reversedSignatureIndex.keys.forEach {
if (it in icModuleReversedFileIndex) error("Duplicate signature $it in both ${icModuleReversedFileIndex[it]!!.originalFileDeserializer.file.path} and in ${file.path}")
icModuleReversedFileIndex[it] = icDeserializer
}
}
}
while (signatureQueue.isNotEmpty()) {
val icFileDeserializer = fileQueue.removeFirst()
val signature = signatureQueue.removeFirst()
val declaration = icFileDeserializer.deserializeDeclaration(signature) ?: continue
icFileDeserializer.injectCarriers(declaration, signature)
icFileDeserializer.mappingsDeserializer(signature, declaration)
// Make sure all members are loaded
if (declaration is IrClass) {
icFileDeserializer.loadClassOrder(signature)?.let {
classToDeclarationSymbols[declaration] = it
}
}
}
irFactory.stageController.withStage(1000) {
for (icDeserializer in icDeserializers) {
val fd = icDeserializer.originalFileDeserializer
val order = icDeserializer.icFileData.order
fd.file.declarations.retainAll { it.isEffectivelyExternal() }
IrLongArrayMemoryReader(order.topLevelSignatures).array.forEach {
val symbolData = icDeserializer.symbolDeserializer.parseSymbolData(it)
val idSig = icDeserializer.symbolDeserializer.deserializeIdSignature(symbolData.signatureId)
// Don't create unbound symbols for top-level declarations we don't need.
if (idSig in icDeserializer.visited) {
val declaration = icDeserializer.deserializeIrSymbol(idSig, symbolData.kind).owner as IrDeclaration
fd.file.declarations += declaration
}
}
}
for ((klass, declarations) in classToDeclarationSymbols.entries) {
irFactory.stageController.unrestrictDeclarationListsAccess {
klass.declarations.clear()
for (ds in declarations) {
klass.declarations += ds.owner as IrDeclaration
}
}
}
}
}
}
@@ -0,0 +1,211 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.backend.common.serialization.DeclarationTable
import org.jetbrains.kotlin.backend.common.serialization.IdSignatureClashTracker
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer
import org.jetbrains.kotlin.backend.common.serialization.signature.PublicIdSignatureComputer
import org.jetbrains.kotlin.ir.IrElement
import org.jetbrains.kotlin.ir.backend.js.JsMapping
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsGlobalDeclarationTable
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrFileSerializer
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.declarations.*
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrBodyBase
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrDeclarationBase
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.expressions.IrExpressionBody
import org.jetbrains.kotlin.ir.serialization.serializeCarriers
import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.util.*
import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
import org.jetbrains.kotlin.ir.visitors.acceptVoid
import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter
import org.jetbrains.kotlin.library.impl.IrMemoryLongArrayWriter
class IcSerializer(
irBuiltIns: IrBuiltIns,
val mappings: JsMapping,
val irFactory: PersistentIrFactory,
val linker: JsIrLinker,
val module: IrModuleFragment
) {
private val globalDeclarationTable = JsGlobalDeclarationTable(irBuiltIns)
fun serializeDeclarations(moduleDeclarations: Iterable<IrDeclaration>): SerializedIcData {
// TODO serialize body carriers and new bodies as well
val moduleDeserializer = linker.moduleDeserializer(module.descriptor)
val fileToDeserializer = moduleDeserializer.fileDeserializers().associateBy { it.file }
val filteredDeclarations = moduleDeclarations.filter {
when {
it.fileOrNull.let { it == null || fileToDeserializer[it] == null } -> false
it is IrFakeOverrideFunction -> it.isBound
it is IrFakeOverrideProperty -> it.isBound
else -> (it.parent as? IrFakeOverrideFunction)?.isBound ?: (it.parent as? IrFakeOverrideProperty)?.isBound ?: true
}
}
val filteredBodies = irFactory.allBodies.groupBy {
(it as? PersistentIrBodyBase<*>)?.let {
if (it.hasContainer) {
it.container.fileOrNull
} else null
}
}
val dataToSerialize = filteredDeclarations.groupBy {
// TODO don't move declarations or effects outside the original file
// TODO Or invent a different mechanism for that
it.file
}
val icData = mutableListOf<SerializedIcDataForFile>()
for (file in fileToDeserializer.keys) {
val fileDeclarations = dataToSerialize[file] ?: emptyList()
val bodies = filteredBodies[file] ?: emptyList()
val fileDeserializer = fileToDeserializer[file]!!
val symbolToSignature = fileDeserializer.symbolDeserializer.deserializedSymbols.entries.associate { (idSig, symbol) -> symbol to idSig }.toMutableMap()
val icDeclarationTable = IcDeclarationTable(globalDeclarationTable, irFactory, 1000000, 1000000, symbolToSignature)
val fileSerializer = JsIrFileSerializer(
linker.messageLogger,
icDeclarationTable,
mutableMapOf(),
skipExpects = true,
icMode = true,
allowNullTypes = true,
allowErrorStatementOrigins = true
)
bodies.forEach { body ->
if (body is IrExpressionBody) {
fileSerializer.serializeIrExpressionBody(body.expression)
} else {
fileSerializer.serializeIrStatementBody(body)
}
}
// Only save newly created declarations
val newDeclarations = fileDeclarations.filter { d ->
d is PersistentIrDeclarationBase<*> && (d.createdOn > 0 || /*d.isFakeOverride ||*/ (d is IrValueParameter || d is IrTypeParameter) && (d.parent as IrDeclaration).isFakeOverride)
}
val serializedCarriers = fileSerializer.serializeCarriers(
fileDeclarations,
bodies,
) { declaration ->
icDeclarationTable.signatureByDeclaration(declaration)
}
val serializedMappings = mappings.state.serializeMappings(fileDeclarations) { symbol ->
fileSerializer.serializeIrSymbol(symbol)
}
val order = storeOrder(file) { symbol ->
fileSerializer.serializeIrSymbol(symbol)
}
val serializedIrFile = fileSerializer.serializeDeclarationsForIC(file, newDeclarations)
icData += SerializedIcDataForFile(
serializedIrFile,
serializedCarriers,
serializedMappings,
order,
)
}
return SerializedIcData(icData)
}
// Returns precomputed signatures for the newly created declarations. Delegates to the default table otherwise.
class IcDeclarationTable(
globalDeclarationTable: JsGlobalDeclarationTable,
val irFactory: PersistentIrFactory,
newLocalIndex: Long,
newScopeIndex: Int,
val existingMappings: MutableMap<IrSymbol, IdSignature>
) : DeclarationTable(globalDeclarationTable) {
override val signaturer: IdSignatureSerializer = IdSignatureSerializerWithForIC(globalDeclarationTable.publicIdSignatureComputer, this, newLocalIndex, newScopeIndex)
override fun signatureByDeclaration(declaration: IrDeclaration): IdSignature {
return existingMappings.getOrPut(declaration.symbol) {
irFactory.declarationSignature(declaration) ?: super.signatureByDeclaration(declaration)
}
}
}
}
class IdSignatureSerializerWithForIC(
publicSignatureBuilder: PublicIdSignatureComputer,
table: DeclarationTable,
localIndexOffset: Long = 0,
scopeIndexOffset: Int = 0,
) : IdSignatureSerializer(
publicSignatureBuilder,
table
) {
init {
localIndex = localIndexOffset
scopeIndex = scopeIndexOffset
}
override fun IrDeclaration.createFileLocalSignature(parentSignature: IdSignature, localIndex: Long): IdSignature {
if (this is IrTypeParameter) {
return IdSignature.GlobalFileLocalSignature(parentSignature, 1000_000_000_000L + index, fileOrNull?.path ?: "")
}
return IdSignature.GlobalFileLocalSignature(parentSignature, localIndex, fileOrNull?.path ?: "")
}
override fun IrDeclaration.createScopeLocalSignature(scopeIndex: Int, description: String): IdSignature {
return IdSignature.GlobalScopeLocalDeclaration(scopeIndex, description, fileOrNull?.path ?: "")
}
}
fun storeOrder(file: IrFile, idSigToLong: (IrSymbol) -> Long): SerializedOrder {
val topLevelSignatures = mutableListOf<Long>()
val containerSignatures = mutableListOf<Long>()
val declarationSignatures = mutableListOf<ByteArray>()
fun IrDeclaration.idSigIndex(): Long = idSigToLong(symbol)
file.declarations.forEach { d ->
topLevelSignatures += d.idSigIndex()
d.acceptVoid(object : IrElementVisitorVoid {
override fun visitElement(element: IrElement) {
element.acceptChildrenVoid(this)
}
override fun visitClass(declaration: IrClass) {
// First element is the container signature
containerSignatures += declaration.idSigIndex()
val declarationIds = declaration.declarations.map { it.idSigIndex() }
declarationSignatures += IrMemoryLongArrayWriter(declarationIds).writeIntoMemory()
super.visitClass(declaration)
}
})
}
return SerializedOrder(
IrMemoryLongArrayWriter(topLevelSignatures).writeIntoMemory(),
IrMemoryLongArrayWriter(containerSignatures).writeIntoMemory(),
IrMemoryArrayWriter(declarationSignatures).writeIntoMemory(),
)
}
@@ -0,0 +1,46 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.ir.declarations.IrFactory
import org.jetbrains.kotlin.ir.declarations.IrTypeParameter
import org.jetbrains.kotlin.ir.symbols.IrFieldSymbol
import org.jetbrains.kotlin.ir.symbols.IrTypeParameterSymbol
import org.jetbrains.kotlin.ir.symbols.impl.IrFieldPublicSymbolImpl
import org.jetbrains.kotlin.ir.symbols.impl.IrTypeParameterPublicSymbolImpl
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.ir.util.IdSignatureComposer
import org.jetbrains.kotlin.ir.util.NameProvider
import org.jetbrains.kotlin.ir.util.SymbolTable
class IcSymbolTable(
signaturer: IdSignatureComposer,
irFactory: IrFactory,
nameProvider: NameProvider = NameProvider.DEFAULT,
) : SymbolTable(
signaturer,
irFactory,
nameProvider,
) {
override fun referenceFieldFromLinker(sig: IdSignature): IrFieldSymbol =
fieldSymbolTable.run {
fieldSymbolTable.referenced(sig) { IrFieldPublicSymbolImpl(sig) }
}
override fun declareGlobalTypeParameter(
sig: IdSignature,
symbolFactory: () -> IrTypeParameterSymbol,
typeParameterFactory: (IrTypeParameterSymbol) -> IrTypeParameter
): IrTypeParameter {
return globalTypeParameterSymbolTable.declare(sig, symbolFactory, typeParameterFactory)
}
override fun referenceTypeParameterFromLinker(sig: IdSignature): IrTypeParameterSymbol {
return scopedTypeParameterSymbolTable.get(sig) ?: globalTypeParameterSymbolTable.referenced(sig) {
IrTypeParameterPublicSymbolImpl(sig)
}
}
}
@@ -0,0 +1,111 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.ic
import org.jetbrains.kotlin.ir.backend.js.SerializedMapping
import org.jetbrains.kotlin.ir.backend.js.SerializedMappings
import org.jetbrains.kotlin.ir.serialization.SerializedCarriers
import org.jetbrains.kotlin.library.SerializedIrFile
import org.jetbrains.kotlin.library.impl.IrArrayMemoryReader
import org.jetbrains.kotlin.library.impl.IrMemoryArrayWriter
import org.jetbrains.kotlin.library.impl.toArray
import java.io.File
import java.nio.charset.Charset
class SerializedIcData(
val files: Collection<SerializedIcDataForFile>,
)
class SerializedIcDataForFile(
val file: SerializedIrFile,
val carriers: SerializedCarriers,
val mappings: SerializedMappings,
val order: SerializedOrder,
)
class SerializedOrder(
val topLevelSignatures: ByteArray,
val containerSignatures: ByteArray,
val declarationSignatures: ByteArray,
)
fun SerializedIcData.writeTo(dir: File) {
if (!dir.exists()) error("Directory doesn't exist: ${dir.absolutePath}")
if (!dir.isDirectory) error("Not a directory: ${dir.absolutePath}")
files.forEach {
val fqnPath = it.file.fqName
val fileId = it.file.path.hashCode().toString(Character.MAX_RADIX)
val irFileDirectory = "ic-$fqnPath.$fileId.file"
val fileDir = File(dir, irFileDirectory)
// TODO: just rewrite?
if (!fileDir.exists()) {
if (!fileDir.mkdirs()) error("Failed to create output dir for file ${fileDir.absolutePath}")
}
// .file
File(fileDir, "file.fileData").writeBytes(it.file.fileData)
File(fileDir, "file.path").writeBytes(it.file.path.toByteArray(Charsets.UTF_8))
File(fileDir, "file.declarations").writeBytes(it.file.declarations)
File(fileDir, "file.types").writeBytes(it.file.types)
File(fileDir, "file.signatures").writeBytes(it.file.signatures)
File(fileDir, "file.strings").writeBytes(it.file.strings)
File(fileDir, "file.bodies").writeBytes(it.file.bodies)
// .carriers
File(fileDir, "carriers.signatures").writeBytes(it.carriers.signatures)
File(fileDir, "carriers.declarationCarriers").writeBytes(it.carriers.declarationCarriers)
File(fileDir, "carriers.bodyCarriers").writeBytes(it.carriers.bodyCarriers)
File(fileDir, "carriers.removedOn").writeBytes(it.carriers.removedOn)
// .mappings
File(fileDir, "mappings.keys").writeBytes(it.mappings.keyBytes())
File(fileDir, "mappings.values").writeBytes(it.mappings.valueBytes())
// .order
File(fileDir, "order.topLevelSignatures").writeBytes(it.order.topLevelSignatures)
File(fileDir, "order.containerSignatures").writeBytes(it.order.containerSignatures)
File(fileDir, "order.declarationSignatures").writeBytes(it.order.declarationSignatures)
}
}
private fun SerializedMappings.keyBytes() = IrMemoryArrayWriter(mappings.map { it.keys }).writeIntoMemory()
private fun SerializedMappings.valueBytes() = IrMemoryArrayWriter(mappings.map { it.values }).writeIntoMemory()
fun File.readIcData(): SerializedIcData {
if (!this.isDirectory) error("Directory doesn't exist: ${this.absolutePath}")
return SerializedIcData(this.listFiles()!!.filter { it.isDirectory}.map { fileDir ->
val file = SerializedIrFile(
fileData = File(fileDir, "file.fileData").readBytes(),
fqName = fileDir.name.split('.').dropLast(2).joinToString(separator = "."),
path = File(fileDir, "file.path").readBytes().toString(Charsets.UTF_8),
types = File(fileDir, "file.types").readBytes(),
signatures = File(fileDir, "file.signatures").readBytes(),
strings = File(fileDir, "file.strings").readBytes(),
bodies = File(fileDir, "file.bodies").readBytes(),
declarations = File(fileDir, "file.declarations").readBytes()
)
val carriers = SerializedCarriers(
signatures = File(fileDir, "carriers.signatures").readBytes(),
declarationCarriers = File(fileDir, "carriers.declarationCarriers").readBytes(),
bodyCarriers = File(fileDir, "carriers.bodyCarriers").readBytes(),
removedOn = File(fileDir, "carriers.removedOn").readBytes(),
)
val mappingKeys = IrArrayMemoryReader(File(fileDir, "mappings.keys").readBytes()).toArray()
val mappingValues = IrArrayMemoryReader(File(fileDir, "mappings.values").readBytes()).toArray()
assert(mappingKeys.size == mappingValues.size)
val mappings = SerializedMappings(mappingKeys.zip(mappingValues).map { (k, v) -> SerializedMapping(k, v) })
val order = SerializedOrder(
topLevelSignatures = File(fileDir, "order.topLevelSignatures").readBytes(),
containerSignatures = File(fileDir, "order.containerSignatures").readBytes(),
declarationSignatures = File(fileDir, "order.declarationSignatures").readBytes(),
)
SerializedIcDataForFile(file, carriers, mappings, order)
})
}
@@ -30,6 +30,8 @@ import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
import org.jetbrains.kotlin.incremental.components.LookupTracker
import org.jetbrains.kotlin.ir.IrBuiltIns
import org.jetbrains.kotlin.ir.ObsoleteDescriptorBasedAPI
import org.jetbrains.kotlin.ir.backend.js.ic.IcSymbolTable
import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrModuleSerializer
import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsManglerDesc
@@ -167,9 +169,9 @@ fun generateKLib(
}
val depsDescriptors =
ModulesStructure(project, MainModule.SourceFiles(files), analyzer, configuration, dependencies, friendDependencies)
ModulesStructure(project, MainModule.SourceFiles(files), analyzer, configuration, dependencies, friendDependencies, EmptyLoweringsCacheProvider)
val allDependencies = depsDescriptors.allDependencies
val (psi2IrContext, hasErrors) = runAnalysisAndPreparePsi2Ir(depsDescriptors, irFactory, errorPolicy)
val (psi2IrContext, hasErrors) = runAnalysisAndPreparePsi2Ir(depsDescriptors, SymbolTable(IdSignatureDescriptor(JsManglerDesc), irFactory, errorPolicy)
val irBuiltIns = psi2IrContext.irBuiltIns
val expectDescriptorToSymbol = mutableMapOf<DeclarationDescriptor, IrSymbol>()
@@ -232,6 +234,7 @@ data class IrModuleInfo(
val symbolTable: SymbolTable,
val deserializer: JsIrLinker,
val moduleFragmentToUniqueName: Map<IrModuleFragment, String>,
val loweredIrLoaded: Set<IrModuleFragment> = emptySet(),
)
private fun sortDependencies(resolvedDependencies: List<KotlinResolvedLibrary>, mapping: Map<KotlinLibrary, ModuleDescriptor>): Collection<KotlinLibrary> {
@@ -244,7 +247,14 @@ private fun sortDependencies(resolvedDependencies: List<KotlinResolvedLibrary>,
}.reversed()
}
@OptIn(ObsoleteDescriptorBasedAPI::class)
interface LoweringsCacheProvider {
fun cacheByPath(path: String): SerializedIcData?
}
object EmptyLoweringsCacheProvider : LoweringsCacheProvider {
override fun cacheByPath(path: String): SerializedIcData? = null
}
fun loadIr(
project: Project,
mainModule: MainModule,
@@ -253,16 +263,20 @@ fun loadIr(
dependencies: Collection<String>,
friendDependencies: Collection<String>,
irFactory: IrFactory,
verifySignatures: Boolean
verifySignatures: Boolean,
loweringsCacheProvider: LoweringsCacheProvider? = null
): IrModuleInfo {
val depsDescriptors = ModulesStructure(project, mainModule, analyzer, configuration, dependencies, friendDependencies)
val depsDescriptors = ModulesStructure(project, mainModule, analyzer, configuration, dependencies, friendDependencies, loweringsCacheProvider ?: EmptyLoweringsCacheProvider)
val errorPolicy = configuration.get(JSConfigurationKeys.ERROR_TOLERANCE_POLICY) ?: ErrorTolerancePolicy.DEFAULT
val messageLogger = configuration.get(IrMessageLogger.IR_MESSAGE_LOGGER) ?: IrMessageLogger.None
val allDependencies = depsDescriptors.allDependencies
val signaturer = IdSignatureDescriptor(JsManglerDesc)
val symbolTable = if (loweringsCacheProvider == null) SymbolTable(signaturer, irFactory) else IcSymbolTable(signaturer, irFactory)
when (mainModule) {
is MainModule.SourceFiles -> {
val (psi2IrContext, _) = runAnalysisAndPreparePsi2Ir(depsDescriptors, irFactory, errorPolicy)
val (psi2IrContext, _) = runAnalysisAndPreparePsi2Ir(depsDescriptors, errorPolicy, symbolTable)
val irBuiltIns = psi2IrContext.irBuiltIns
val symbolTable = psi2IrContext.symbolTable
val feContext = psi2IrContext.run {
@@ -271,6 +285,16 @@ fun loadIr(
val moduleFragmentToUniqueName = mutableMapOf<IrModuleFragment, String>()
val irLinker =
JsIrLinker(psi2IrContext.moduleDescriptor, messageLogger, irBuiltIns, symbolTable, feContext, null)
JsIrLinker(
psi2IrContext.moduleDescriptor,
messageLogger,
irBuiltIns,
symbolTable,
feContext,
null,
depsDescriptors.loweredIcData,
loweringsCacheProvider != null
)
val deserializedModuleFragments = sortDependencies(allDependencies, depsDescriptors.descriptors).map { klib ->
irLinker.deserializeIrModuleHeader(
depsDescriptors.getModuleDescriptor(klib),
@@ -301,7 +325,8 @@ fun loadIr(
(irBuiltIns as IrBuiltInsOverDescriptors).knownBuiltins.forEach { it.acceptVoid(mangleChecker) }
}
return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName)
return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName,
depsDescriptors.modulesWithCaches(deserializedModuleFragments))
}
is MainModule.Klib -> {
val mainModuleLib = depsDescriptors.allDependencies.find { it.library.libraryFile.canonicalPath == mainModule.libPath }?.library
@@ -313,8 +338,32 @@ fun loadIr(
val typeTranslator =
TypeTranslatorImpl(symbolTable, depsDescriptors.compilerConfiguration.languageVersionSettings, moduleDescriptor)
val irBuiltIns = IrBuiltInsOverDescriptors(moduleDescriptor.builtIns, typeTranslator, symbolTable)
val loweredIcData = if (loweringsCacheProvider == null) emptyMap() else {
val result = mutableMapOf<ModuleDescriptor, SerializedIcData>()
for (lib in depsDescriptors.moduleDependencies.keys) {
val path = lib.libraryFile.absolutePath
val icData = loweringsCacheProvider.cacheByPath(path)
if (icData != null) {
result[depsDescriptors.getModuleDescriptor(lib)] = icData
}
}
result
}
val irLinker =
JsIrLinker(null, messageLogger, irBuiltIns, symbolTable, null, null)
JsIrLinker(
null,
messageLogger,
irBuiltIns,
symbolTable,
null,
null,
loweredIcData,
loweringsCacheProvider != null
)
val moduleFragmentToUniqueName = mutableMapOf<IrModuleFragment, String>()
@@ -338,22 +387,22 @@ fun loadIr(
ExternalDependenciesGenerator(symbolTable, listOf(irLinker)).generateUnboundSymbolsAsDependencies()
irLinker.postProcess()
return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName)
return IrModuleInfo(moduleFragment, deserializedModuleFragments, irBuiltIns, symbolTable, irLinker, moduleFragmentToUniqueName,
depsDescriptors.modulesWithCaches(deserializedModuleFragments))
}
}
}
private fun runAnalysisAndPreparePsi2Ir(
depsDescriptors: ModulesStructure,
irFactory: IrFactory,
errorIgnorancePolicy: ErrorTolerancePolicy
errorIgnorancePolicy: ErrorTolerancePolicy,
symbolTable: SymbolTable,
): Pair<GeneratorContext, Boolean> {
val analysisResult = depsDescriptors.runAnalysis(errorIgnorancePolicy)
val psi2Ir = Psi2IrTranslator(
depsDescriptors.compilerConfiguration.languageVersionSettings,
Psi2IrConfiguration(errorIgnorancePolicy.allowErrors)
)
val symbolTable = SymbolTable(IdSignatureDescriptor(JsManglerDesc), irFactory)
return psi2Ir.createGeneratorContext(
analysisResult.moduleDescriptor,
analysisResult.bindingContext,
@@ -426,7 +475,8 @@ private class ModulesStructure(
private val analyzer: AbstractAnalyzerWithCompilerReport,
val compilerConfiguration: CompilerConfiguration,
dependencies: Collection<String>,
friendDependenciesPaths: Collection<String>
friendDependenciesPaths: Collection<String>,
private val loweringsCacheProvider: LoweringsCacheProvider
) {
val allDependencies = jsResolveLibraries(
dependencies,
@@ -497,6 +547,8 @@ private class ModulesStructure(
// TODO: these are roughly equivalent to KlibResolvedModuleDescriptorsFactoryImpl. Refactor me.
val descriptors = mutableMapOf<KotlinLibrary, ModuleDescriptorImpl>()
val loweredIcData = mutableMapOf<ModuleDescriptor, SerializedIcData>()
fun getModuleDescriptor(current: KotlinLibrary): ModuleDescriptorImpl = descriptors.getOrPut(current) {
val isBuiltIns = current.unresolvedDependencies.isEmpty()
@@ -513,9 +565,18 @@ private class ModulesStructure(
val dependencies = moduleDependencies.getValue(current).map { getModuleDescriptor(it) }
md.setDependencies(listOf(md) + dependencies)
loweringsCacheProvider.cacheByPath(current.libraryFile.absolutePath)?.let { icData ->
loweredIcData[md] = icData
}
md
}
fun modulesWithCaches(allModules: Iterable<IrModuleFragment>): Set<IrModuleFragment> {
return allModules.filter { it.descriptor in loweredIcData }.toSet()
}
val builtInModuleDescriptor =
if (builtInsDep != null)
getModuleDescriptor(builtInsDep.library)
@@ -0,0 +1,66 @@
/*
* Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
* Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
*/
package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializer
import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializerWithBuiltIns
import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData
import org.jetbrains.kotlin.backend.common.serialization.knownBuiltins
import org.jetbrains.kotlin.ir.declarations.IrSimpleFunction
import org.jetbrains.kotlin.ir.declarations.IrSymbolOwner
import org.jetbrains.kotlin.ir.declarations.IrTypeParametersContainer
import org.jetbrains.kotlin.ir.descriptors.IrAbstractFunctionFactory
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.symbols.IrSymbol
import org.jetbrains.kotlin.ir.util.IdSignature
class IrIcModuleDeserializerWithBuiltIns(
builtIns: IrBuiltIns,
functionFactory: IrAbstractFunctionFactory,
delegate: IrModuleDeserializer,
) : IrModuleDeserializerWithBuiltIns(builtIns, functionFactory, delegate) {
override fun additionalBuiltIns(builtIns: IrBuiltIns): Map<IdSignature, IrSymbol> {
val result = mutableMapOf<IdSignature, IrSymbol>()
builtIns.knownBuiltins.forEach {
val symbol = (it as IrSymbolOwner).symbol
val declaration = symbol.owner
if (declaration is IrSimpleFunction) {
declaration.typeParameters.forEachIndexed { i, tp ->
result[IdSignature.GlobalFileLocalSignature(symbol.signature!!, 1000_000_000_000L + i, "")] = tp.symbol
}
}
}
return result
}
override fun checkIsFunctionInterface(idSig: IdSignature): Boolean {
if (idSig is IdSignature.GlobalFileLocalSignature) return checkIsFunctionInterface(idSig.container)
return super.checkIsFunctionInterface(idSig)
}
override fun contains(idSig: IdSignature): Boolean {
return super.contains(idSig) || idSig is IdSignature.GlobalFileLocalSignature && checkIsFunctionInterface(idSig.container)
}
override fun resolveFunctionalInterface(idSig: IdSignature, symbolKind: BinarySymbolData.SymbolKind): IrSymbol {
if (idSig is IdSignature.GlobalFileLocalSignature) {
val containerSymbolKind = when (idSig.container.asPublic()!!.nameSegments.size) {
1 -> BinarySymbolData.SymbolKind.CLASS_SYMBOL
3 -> BinarySymbolData.SymbolKind.FUNCTION_SYMBOL
else -> error("Cannot infer symbolKind")
}
val declaration = resolveFunctionalInterface(idSig.container, containerSymbolKind).owner as IrTypeParametersContainer
return declaration.typeParameters[(idSig.id - 1000_000_000_000L).toInt()].symbol
}
return super.resolveFunctionalInterface(idSig, symbolKind)
}
}
@@ -14,7 +14,7 @@ import org.jetbrains.kotlin.ir.declarations.IrTypeParameter
import org.jetbrains.kotlin.ir.util.IdSignature
import org.jetbrains.kotlin.ir.util.render
class JsUniqIdClashTracker : IdSignatureClashTracker {
class JsUniqIdClashTracker() : IdSignatureClashTracker {
private val committedIdSignatures = mutableMapOf<IdSignature, IrDeclaration>()
override fun commit(declaration: IrDeclaration, signature: IdSignature) {
@@ -34,8 +34,8 @@ class JsUniqIdClashTracker : IdSignatureClashTracker {
}
}
class JsGlobalDeclarationTable(builtIns: IrBuiltIns) :
GlobalDeclarationTable(JsManglerIr, JsUniqIdClashTracker()) {
class JsGlobalDeclarationTable(builtIns: IrBuiltIns, tracker: IdSignatureClashTracker = JsUniqIdClashTracker()) :
GlobalDeclarationTable(JsManglerIr, tracker) {
init {
loadKnownBuiltins(builtIns)
}
@@ -24,6 +24,8 @@ class JsIrFileSerializer(
skipExpects: Boolean,
bodiesOnlyForInlines: Boolean = false,
icMode: Boolean = false,
allowNullTypes: Boolean = false,
allowErrorStatementOrigins: Boolean = false,
) : IrFileSerializer(
messageLogger,
declarationTable,
@@ -32,6 +34,8 @@ class JsIrFileSerializer(
bodiesOnlyForInlines = bodiesOnlyForInlines,
skipExpects = skipExpects,
skipMutableState = icMode,
allowNullTypes = allowNullTypes,
allowErrorStatementOrigins = allowErrorStatementOrigins,
) {
companion object {
private val JS_EXPORT_FQN = FqName("kotlin.js.JsExport")
@@ -7,10 +7,18 @@ package org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir
import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideBuilder
import org.jetbrains.kotlin.backend.common.serialization.*
import org.jetbrains.kotlin.backend.common.serialization.signature.IdSignatureSerializer
import org.jetbrains.kotlin.descriptors.ModuleDescriptor
import org.jetbrains.kotlin.ir.backend.js.JsMapping
import org.jetbrains.kotlin.ir.backend.js.ic.IcModuleDeserializer
import org.jetbrains.kotlin.ir.backend.js.ic.IdSignatureSerializerWithForIC
import org.jetbrains.kotlin.ir.backend.js.ic.SerializedIcData
import org.jetbrains.kotlin.ir.IrBuiltIns
import org.jetbrains.kotlin.ir.builders.TranslationPluginContext
import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
import org.jetbrains.kotlin.ir.declarations.persistent.PersistentIrFactory
import org.jetbrains.kotlin.ir.descriptors.IrAbstractFunctionFactory
import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns
import org.jetbrains.kotlin.ir.types.IrTypeSystemContextImpl
import org.jetbrains.kotlin.ir.util.IrMessageLogger
import org.jetbrains.kotlin.ir.util.ReferenceSymbolTable
@@ -24,10 +32,15 @@ import org.jetbrains.kotlin.library.containsErrorCode
class JsIrLinker(
private val currentModule: ModuleDescriptor?, messageLogger: IrMessageLogger, builtIns: IrBuiltIns, symbolTable: SymbolTable,
override val translationPluginContext: TranslationPluginContext?,
private val icData: ICData? = null
private val icData: ICData? = null,
private val loweredIcData: Map<ModuleDescriptor, SerializedIcData> = emptyMap(),
private val useGlobalSignatures: Boolean = false,
) : KotlinIrLinker(currentModule, messageLogger, builtIns, symbolTable, emptyList()) {
override val fakeOverrideBuilder = FakeOverrideBuilder(this, symbolTable, JsManglerIr, IrTypeSystemContextImpl(builtIns))
override val fakeOverrideBuilder = FakeOverrideBuilder(this, symbolTable, JsManglerIr, IrTypeSystemContextImpl(builtIns),
signatureSerializerFactory = { publicSignatureBuilder, table ->
if (useGlobalSignatures) IdSignatureSerializerWithForIC(publicSignatureBuilder, table) else IdSignatureSerializer(publicSignatureBuilder, table)
})
override fun isBuiltInModule(moduleDescriptor: ModuleDescriptor): Boolean =
moduleDescriptor === moduleDescriptor.builtIns.builtInsModule
@@ -35,13 +48,43 @@ class JsIrLinker(
private val IrLibrary.libContainsErrorCode: Boolean
get() = this is KotlinLibrary && this.containsErrorCode
override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: KotlinLibrary?, strategy: DeserializationStrategy): IrModuleDeserializer = klib?.let { lib ->
JsModuleDeserializer(moduleDescriptor, lib, strategy, lib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, lib.libContainsErrorCode)
} ?: error("Expecting kotlin library")
override fun createModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary?, strategy: DeserializationStrategy): IrModuleDeserializer {
require(klib != null) { "Expecting kotlin library" }
loweredIcData[moduleDescriptor]?.let { loweredIcData ->
return IcModuleDeserializer(
symbolTable.irFactory as PersistentIrFactory,
mapping,
this,
loweredIcData,
moduleDescriptor,
klib,
strategy,
containsErrorCode = klib.libContainsErrorCode,
useGlobalSignatures = useGlobalSignatures
)
}
return klib?.let { lib ->
JsModuleDeserializer(moduleDescriptor, lib, strategy, lib.versions.abiVersion ?: KotlinAbiVersion.CURRENT, lib.libContainsErrorCode)
} ?: error("Expecting kotlin library")
}
val mapping: JsMapping by lazy { JsMapping(symbolTable.irFactory) }
private inner class JsModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary, strategy: DeserializationStrategy, libraryAbiVersion: KotlinAbiVersion, allowErrorCode: Boolean) :
BasicIrModuleDeserializer(this, moduleDescriptor, klib, strategy, libraryAbiVersion, allowErrorCode)
private inner class JsModuleDeserializer(moduleDescriptor: ModuleDescriptor, klib: IrLibrary, strategy: DeserializationStrategy, allowErrorCode: Boolean) :
BasicIrModuleDeserializer(this, moduleDescriptor, klib, strategy, libraryAbiVersion, allowErrorCode, useGlobalSignatures)
override fun maybeWrapWithBuiltInAndInit(
moduleDescriptor: ModuleDescriptor,
moduleDeserializer: IrModuleDeserializer
): IrModuleDeserializer {
return if (isBuiltInModule(moduleDescriptor)) {
if (useGlobalSignatures) {
IrIcModuleDeserializerWithBuiltIns(builtIns, functionalInterfaceFactory, moduleDeserializer)
} else {
IrModuleDeserializerWithBuiltIns(builtIns, functionalInterfaceFactory, moduleDeserializer)
}
} else moduleDeserializer
}
override fun createCurrentModuleDeserializer(moduleFragment: IrModuleFragment, dependencies: Collection<IrModuleDeserializer>): IrModuleDeserializer {
val currentModuleDeserializer = super.createCurrentModuleDeserializer(moduleFragment, dependencies)
@@ -58,6 +101,20 @@ class JsIrLinker(
.map { it.moduleFragment }
.filter { it.descriptor !== currentModule }
fun moduleDeserializer(moduleDescriptor: ModuleDescriptor): IrModuleDeserializer {
return deserializersForModules[moduleDescriptor] ?: error("Deserializer for $moduleDescriptor not found")
}
fun loadIcIr(preprocess: (IrModuleFragment) -> Unit) {
deserializersForModules.values.forEach {
if (it.moduleDescriptor in loweredIcData) {
preprocess(it.moduleFragment)
}
it.postProcess()
}
}
class JsFePluginContext(
override val moduleDescriptor: ModuleDescriptor,
override val symbolTable: ReferenceSymbolTable,