diff --git a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/linkage/KotlinIrLinkerInternalException.kt b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/linkage/KotlinIrLinkerInternalException.kt index 39948d0416d..bc2325d7c4f 100644 --- a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/linkage/KotlinIrLinkerInternalException.kt +++ b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/linkage/KotlinIrLinkerInternalException.kt @@ -5,4 +5,5 @@ package org.jetbrains.kotlin.ir.linkage -object KotlinIrLinkerInternalException : Exception("Kotlin IR Linker exception") \ No newline at end of file +// Used to terminate linking process. Detailed linkage errors are reported separately to IrMessageLogger. +class KotlinIrLinkerInternalException : Exception("Kotlin IR Linker exception") diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/BasicIrModuleDeserializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/BasicIrModuleDeserializer.kt index 761c4e72c9d..4e0806e4cb2 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/BasicIrModuleDeserializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/BasicIrModuleDeserializer.kt @@ -115,8 +115,7 @@ abstract class BasicIrModuleDeserializer( strategy.needBodies, allowErrorNodes, strategy.inlineBodies, - moduleDeserializer, - linker::handleNoModuleDeserializerFound, + moduleDeserializer ) fileToDeserializerMap[file] = fileDeserializationState.fileDeserializer diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrDeclarationDeserializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrDeclarationDeserializer.kt index e4a1ef90c36..881acfa3cd5 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrDeclarationDeserializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrDeclarationDeserializer.kt @@ -8,6 +8,8 @@ package org.jetbrains.kotlin.backend.common.serialization import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideBuilder import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideClassFilter import org.jetbrains.kotlin.backend.common.serialization.encodings.* +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkErrorNodesAllowed +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.checkSymbolType import org.jetbrains.kotlin.backend.common.serialization.proto.IrDeclaration.DeclaratorCase.* import org.jetbrains.kotlin.backend.common.serialization.proto.IrType.KindCase.* import org.jetbrains.kotlin.descriptors.InlineClassRepresentation @@ -124,13 +126,12 @@ class IrDeclarationDeserializer( } private fun deserializeSimpleType(proto: ProtoSimpleType): IrSimpleType { - val symbol = deserializeIrSymbolAndRemap(proto.classifier) as? IrClassifierSymbol - ?: error("could not convert sym to ClassifierSymbol") + val symbol = checkSymbolType(deserializeIrSymbolAndRemap(proto.classifier)) val arguments = proto.argumentList.map { deserializeIrTypeArgument(it) } val annotations = deserializeAnnotations(proto.annotationList) - val result: IrSimpleType = IrSimpleTypeImpl( + return IrSimpleTypeImpl( null, symbol, proto.hasQuestionMark, @@ -138,16 +139,11 @@ class IrDeclarationDeserializer( annotations, if (proto.hasAbbreviation()) deserializeTypeAbbreviation(proto.abbreviation) else null ) - return result - } private fun deserializeTypeAbbreviation(proto: ProtoTypeAbbreviation): IrTypeAbbreviation = IrTypeAbbreviationImpl( - deserializeIrSymbolAndRemap(proto.typeAlias).let { - it as? IrTypeAliasSymbol - ?: error("IrTypeAliasSymbol expected: $it") - }, + checkSymbolType(deserializeIrSymbolAndRemap(proto.typeAlias)), proto.hasQuestionMark, proto.argumentList.map { deserializeIrTypeArgument(it) }, deserializeAnnotations(proto.annotationList) @@ -159,7 +155,7 @@ class IrDeclarationDeserializer( } private fun deserializeErrorType(proto: ProtoErrorType): IrErrorType { - require(allowErrorNodes) { "IrErrorType found but error code is not allowed" } + checkErrorNodesAllowed(allowErrorNodes) val annotations = deserializeAnnotations(proto.annotationList) return IrErrorTypeImpl(null, annotations, Variance.INVARIANT) } @@ -273,7 +269,7 @@ class IrDeclarationDeserializer( val result = symbolTable.run { if (isGlobal) { val p = symbolDeserializer.deserializeIrSymbolToDeclare(proto.base.symbol) - val symbol = p.first as IrTypeParameterSymbol + val symbol = checkSymbolType(p.first) sig = p.second declareGlobalTypeParameter(sig, { symbol }, factory) } else { @@ -297,7 +293,7 @@ class IrDeclarationDeserializer( val nameAndType = BinaryNameAndType.decode(proto.nameType) irFactory.createValueParameter( startOffset, endOffset, origin, - symbol as IrValueParameterSymbol, + checkSymbolType(symbol), deserializeName(nameAndType.nameIndex), index, deserializeIrType(nameAndType.typeIndex), @@ -314,11 +310,11 @@ class IrDeclarationDeserializer( fun deserializeIrClass(proto: ProtoClass): IrClass = withDeserializedIrDeclarationBase(proto.base) { symbol, signature, startOffset, endOffset, origin, fcode -> + checkSymbolType(symbol) + if (allowRedeclaration && symbol.isBound) return symbol.owner + val flags = ClassFlags.decode(fcode) - - if (allowRedeclaration && symbol.isBound) return symbol.owner as IrClass - - symbolTable.declareClass(signature, { symbol as IrClassSymbol }) { + symbolTable.declareClass(signature, { symbol }) { irFactory.createClass( startOffset, endOffset, origin, it, @@ -379,7 +375,7 @@ class IrDeclarationDeserializer( private fun deserializeIrTypeAlias(proto: ProtoTypeAlias): IrTypeAlias = withDeserializedIrDeclarationBase(proto.base) { symbol, uniqId, startOffset, endOffset, origin, fcode -> - require(symbol is IrTypeAliasSymbol) + checkSymbolType(symbol) symbolTable.declareTypeAlias(uniqId, { symbol }) { val flags = TypeAliasFlags.decode(fcode) val nameType = BinaryNameAndType.decode(proto.nameType) @@ -400,7 +396,7 @@ class IrDeclarationDeserializer( } private fun deserializeErrorDeclaration(proto: ProtoErrorDeclaration): IrErrorDeclaration { - require(allowErrorNodes) { "IrErrorDeclaration found but error code is not allowed" } + checkErrorNodesAllowed(allowErrorNodes) val coordinates = BinaryCoordinates.decode(proto.coordinates) return irFactory.createErrorDeclaration(coordinates.startOffset, coordinates.endOffset).also { it.parent = currentParent @@ -528,7 +524,7 @@ class IrDeclarationDeserializer( block: (IrFunctionSymbol, IdSignature, Int, Int, IrDeclarationOrigin, Long) -> T ): T = withDeserializedIrDeclarationBase(proto.base) { symbol, idSig, startOffset, endOffset, origin, fcode -> symbolTable.withScope(symbol) { - block(symbol as IrFunctionSymbol, idSig, startOffset, endOffset, origin, fcode).usingParent { + block(checkSymbolType(symbol), idSig, startOffset, endOffset, origin, fcode).usingParent { if (!skipMutableState) { typeParameters = deserializeTypeParameters(proto.typeParameterList, false) val nameType = BinaryNameAndType.decode(proto.nameType) @@ -553,9 +549,11 @@ class IrDeclarationDeserializer( internal fun deserializeIrFunction(proto: ProtoFunction): IrSimpleFunction { return withDeserializedIrFunctionBase(proto.base) { symbol, idSig, startOffset, endOffset, origin, fcode -> + checkSymbolType(symbol) + if (allowRedeclaration && symbol.isBound) return symbol.owner + val flags = FunctionFlags.decode(fcode) - if (allowRedeclaration && symbol.isBound) return symbol.owner as IrSimpleFunction - symbolTable.declareSimpleFunction(idSig, { symbol as IrSimpleFunctionSymbol }) { + symbolTable.declareSimpleFunction(idSig, { symbol }) { val nameType = BinaryNameAndType.decode(proto.base.nameType) irFactory.createFunction( startOffset, endOffset, origin, @@ -574,18 +572,21 @@ class IrDeclarationDeserializer( flags.isFakeOverride ) }.apply { - overriddenSymbols = proto.overriddenList.map { deserializeIrSymbolAndRemap(it) as IrSimpleFunctionSymbol } + overriddenSymbols = proto.overriddenList.map { checkSymbolType(deserializeIrSymbolAndRemap(it)) } } } } fun deserializeIrVariable(proto: ProtoVariable): IrVariable = withDeserializedIrDeclarationBase(proto.base) { symbol, _, startOffset, endOffset, origin, fcode -> + checkSymbolType(symbol) + val flags = LocalVariableFlags.decode(fcode) val nameType = BinaryNameAndType.decode(proto.nameType) - (if (allowRedeclaration && symbol.isBound) symbol.owner as IrVariable else IrVariableImpl( + + (if (allowRedeclaration && symbol.isBound) symbol.owner else IrVariableImpl( startOffset, endOffset, origin, - symbol as IrVariableSymbol, + symbol, deserializeName(nameType.nameIndex), deserializeIrType(nameType.typeIndex), flags.isVar, @@ -599,7 +600,7 @@ class IrDeclarationDeserializer( private fun deserializeIrEnumEntry(proto: ProtoEnumEntry): IrEnumEntry = withDeserializedIrDeclarationBase(proto.base) { symbol, uniqId, startOffset, endOffset, origin, _ -> - symbolTable.declareEnumEntry(uniqId, { symbol as IrEnumEntrySymbol }) { + symbolTable.declareEnumEntry(uniqId, { checkSymbolType(symbol) }) { irFactory.createEnumEntry(startOffset, endOffset, origin, it, deserializeName(proto.name)) }.apply { if (!skipMutableState) { @@ -613,14 +614,14 @@ class IrDeclarationDeserializer( private fun deserializeIrAnonymousInit(proto: ProtoAnonymousInit): IrAnonymousInitializer = withDeserializedIrDeclarationBase(proto.base) { symbol, _, startOffset, endOffset, origin, _ -> - irFactory.createAnonymousInitializer(startOffset, endOffset, origin, symbol as IrAnonymousInitializerSymbol).apply { + irFactory.createAnonymousInitializer(startOffset, endOffset, origin, checkSymbolType(symbol)).apply { body = deserializeStatementBody(proto.body) as IrBlockBody } } private fun deserializeIrConstructor(proto: ProtoConstructor): IrConstructor = withDeserializedIrFunctionBase(proto.base) { symbol, idSig, startOffset, endOffset, origin, fcode -> - require(symbol is IrConstructorSymbol) + checkSymbolType(symbol) val flags = FunctionFlags.decode(fcode) val nameType = BinaryNameAndType.decode(proto.base.nameType) symbolTable.declareConstructor(idSig, { symbol }) { @@ -641,7 +642,7 @@ class IrDeclarationDeserializer( private fun deserializeIrField(proto: ProtoField): IrField = withDeserializedIrDeclarationBase(proto.base) { symbol, uniqId, startOffset, endOffset, origin, fcode -> - require(symbol is IrFieldSymbol) + checkSymbolType(symbol) val nameType = BinaryNameAndType.decode(proto.nameType) val type = deserializeIrType(nameType.typeIndex) val flags = FieldFlags.decode(fcode) @@ -672,11 +673,14 @@ class IrDeclarationDeserializer( private fun deserializeIrLocalDelegatedProperty(proto: ProtoLocalDelegatedProperty): IrLocalDelegatedProperty = withDeserializedIrDeclarationBase(proto.base) { symbol, _, startOffset, endOffset, origin, fcode -> + checkSymbolType(symbol) + val flags = LocalVariableFlags.decode(fcode) val nameAndType = BinaryNameAndType.decode(proto.nameType) - val prop = if (allowRedeclaration && symbol.isBound) symbol.owner as IrLocalDelegatedProperty else irFactory.createLocalDelegatedProperty( + + val prop = if (allowRedeclaration && symbol.isBound) symbol.owner else irFactory.createLocalDelegatedProperty( startOffset, endOffset, origin, - symbol as IrLocalDelegatedPropertySymbol, + symbol, deserializeName(nameAndType.nameIndex), deserializeIrType(nameAndType.typeIndex), flags.isVar @@ -696,7 +700,7 @@ class IrDeclarationDeserializer( private fun deserializeIrProperty(proto: ProtoProperty): IrProperty = withDeserializedIrDeclarationBase(proto.base) { symbol, uniqId, startOffset, endOffset, origin, fcode -> - require(symbol is IrPropertySymbol) + checkSymbolType(symbol) val flags = PropertyFlags.decode(fcode) val prop = if (allowRedeclaration && symbol.isBound) symbol.owner else symbolTable.declareProperty(uniqId, { symbol }) { irFactory.createProperty( diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt index 5efb48f7bbb..11d5500fcd9 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/IrFileDeserializer.kt @@ -5,13 +5,11 @@ package org.jetbrains.kotlin.backend.common.serialization -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrFile import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.declarations.impl.IrFileImpl -import org.jetbrains.kotlin.ir.declarations.path import org.jetbrains.kotlin.ir.symbols.impl.IrFileSymbolImpl import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl @@ -65,8 +63,7 @@ class FileDeserializationState( deserializeBodies: Boolean, allowErrorNodes: Boolean, deserializeInlineFunctions: Boolean, - moduleDeserializer: IrModuleDeserializer, - handleNoModuleDeserializerFound: (IdSignature, ModuleDescriptor, Collection) -> IrModuleDeserializer, + moduleDeserializer: IrModuleDeserializer ) { val symbolDeserializer = @@ -78,12 +75,13 @@ class FileDeserializationState( ) { idSig, symbolKind -> val topLevelSig = idSig.topLevelSignature() - val actualModuleDeserializer = - moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) ?: handleNoModuleDeserializerFound( - idSig, - moduleDeserializer.moduleDescriptor, - moduleDeserializer.moduleDependencies - ) + val actualModuleDeserializer = moduleDeserializer.findModuleDeserializerForTopLevelId(topLevelSig) + ?: run { + // The symbol might be gone in newer version of dependency KLIB. Then the KLIB that was compiled against + // the older version of dependency KLIB will still have a reference to non-existing symbol. And the linker will have to + // handle such situation appropriately. See KT-41378. + linker.handleSignatureIdNotFoundInModuleWithDependencies(idSig, moduleDeserializer) + } actualModuleDeserializer.deserializeIrSymbol(idSig, symbolKind) } diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt index 2095eb42274..6a18a77ae2b 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt @@ -8,6 +8,7 @@ package org.jetbrains.kotlin.backend.common.serialization import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideBuilder import org.jetbrains.kotlin.backend.common.overrides.FileLocalAwareLinker import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.* import org.jetbrains.kotlin.descriptors.CallableMemberDescriptor import org.jetbrains.kotlin.descriptors.DeclarationDescriptor import org.jetbrains.kotlin.descriptors.ModuleDescriptor @@ -16,7 +17,6 @@ import org.jetbrains.kotlin.ir.builders.TranslationPluginContext import org.jetbrains.kotlin.ir.declarations.IrDeclaration import org.jetbrains.kotlin.ir.declarations.IrModuleFragment import org.jetbrains.kotlin.ir.linkage.IrDeserializer -import org.jetbrains.kotlin.ir.linkage.KotlinIrLinkerInternalException import org.jetbrains.kotlin.ir.symbols.* import org.jetbrains.kotlin.ir.util.IdSignature import org.jetbrains.kotlin.ir.util.IrMessageLogger @@ -53,33 +53,23 @@ abstract class KotlinIrLinker( private lateinit var linkerExtensions: Collection - public open fun handleNoModuleDeserializerFound(idSignature: IdSignature, currentModule: ModuleDescriptor, dependencies: Collection): IrModuleDeserializer { - val message = buildString { - append("Module ${currentModule.name} has reference $idSignature, unfortunately neither itself nor its dependencies ") - dependencies.joinTo(this, "\n\t", "[\n\t", "\n]") { it.moduleDescriptor.name.asString() } - append(" contain this declaration") - append("\n") - append("Please check that project configuration is correct and has required dependencies.") - } - messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) + protected open val userVisibleIrModulesSupport: UserVisibleIrModulesSupport = UserVisibleIrModulesSupport.DEFAULT - throw KotlinIrLinkerInternalException + fun handleSignatureIdNotFoundInModuleWithDependencies( + idSignature: IdSignature, + moduleDeserializer: IrModuleDeserializer + ): IrModuleDeserializer { + throw SignatureIdNotFoundInModuleWithDependencies( + idSignature = idSignature, + problemModuleDeserializer = moduleDeserializer, + allModuleDeserializers = deserializersForModules.values, + userVisibleIrModulesSupport = userVisibleIrModulesSupport + ).raiseIssue(messageLogger) } - public open fun resolveModuleDeserializer(module: ModuleDescriptor, signature: IdSignature?): IrModuleDeserializer { - return deserializersForModules[module.name.asString()] ?: run { - val message = buildString { - append("Could not load module ") - append(module) - signature?.let { - append("; It was an attempt to find deserializer for ") - append(it) - } - } - messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) - - throw KotlinIrLinkerInternalException - } + fun resolveModuleDeserializer(module: ModuleDescriptor, idSignature: IdSignature?): IrModuleDeserializer { + return deserializersForModules[module.name.asString()] + ?: throw NoDeserializerForModule(module.name, idSignature).raiseIssue(messageLogger) } protected abstract fun createModuleDeserializer( @@ -153,7 +143,11 @@ abstract class KotlinIrLinker( } if (!symbol.isBound) { - findDeserializedDeclarationForSymbol(symbol) ?: tryResolveCustomDeclaration(symbol) ?: return null + try { + findDeserializedDeclarationForSymbol(symbol) ?: tryResolveCustomDeclaration(symbol) ?: return null + } catch (e: IrSymbolTypeMismatchException) { + throw SymbolTypeMismatch(e, deserializersForModules.values, userVisibleIrModulesSupport).raiseIssue(messageLogger) + } } // TODO: we do have serializations for those, but let's just create a stub for now. diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/IrDeserializationExceptions.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/IrDeserializationExceptions.kt new file mode 100644 index 00000000000..941f6e559e7 --- /dev/null +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/IrDeserializationExceptions.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.common.serialization.linkerissues + +import org.jetbrains.kotlin.ir.declarations.IrAnnotationContainer +import org.jetbrains.kotlin.ir.symbols.IrSymbol +import kotlin.contracts.ExperimentalContracts +import kotlin.contracts.contract + +sealed class IrDeserializationException(message: String) : Exception(message) { + override val message: String get() = super.message!! +} + +class IrSymbolTypeMismatchException( + val expected: Class, + val actual: IrSymbol +) : IrDeserializationException("The symbol of unexpected type encountered during IR deserialization: ${actual::class.java.simpleName}, ${actual.signature?.render() ?: actual.toString()}. ${expected.simpleName} is expected.") + +class IrDisallowedErrorNode( + clazz: Class +) : IrDeserializationException("${clazz::class.java.simpleName} found but error nodes are not allowed.") + +@OptIn(ExperimentalContracts::class) +internal inline fun checkSymbolType(symbol: IrSymbol): T { + contract { + returns() implies (symbol is T) + } + + if (symbol !is T) throw IrSymbolTypeMismatchException(T::class.java, symbol) else return symbol +} + +@OptIn(ExperimentalContracts::class) +internal inline fun checkErrorNodesAllowed(errorNodesAllowed: Boolean) { + contract { + returns() implies errorNodesAllowed + } + if (!errorNodesAllowed) throw IrDisallowedErrorNode(T::class.java) +} diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/KotlinIrLinkerIssues.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/KotlinIrLinkerIssues.kt new file mode 100644 index 00000000000..4cfe783b27e --- /dev/null +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/KotlinIrLinkerIssues.kt @@ -0,0 +1,536 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.common.serialization.linkerissues + +import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializer +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.PotentialConflictKind.* +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.PotentialConflictKind.Companion.mostSignificantConflictKind +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.PotentialConflictReason.Companion.mostSignificantConflictReasons +import org.jetbrains.kotlin.ir.linkage.KotlinIrLinkerInternalException +import org.jetbrains.kotlin.ir.util.IdSignature +import org.jetbrains.kotlin.ir.util.IrMessageLogger +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.utils.ResolvedDependency +import org.jetbrains.kotlin.utils.ResolvedDependencyId +import org.jetbrains.kotlin.utils.ResolvedDependencyVersion +import kotlin.Comparator + +abstract class KotlinIrLinkerIssue { + protected abstract val message: String + + fun raiseIssue(messageLogger: IrMessageLogger): KotlinIrLinkerInternalException { + messageLogger.report(IrMessageLogger.Severity.ERROR, message, null) + throw KotlinIrLinkerInternalException() + } +} + +class SignatureIdNotFoundInModuleWithDependencies( + idSignature: IdSignature, + problemModuleDeserializer: IrModuleDeserializer, + allModuleDeserializers: Collection, + userVisibleIrModulesSupport: UserVisibleIrModulesSupport +) : KotlinIrLinkerIssue() { + override val message = buildString { + val allModules = userVisibleIrModulesSupport.getUserVisibleModules(allModuleDeserializers) + + val problemModuleId = userVisibleIrModulesSupport.getProblemModuleId(problemModuleDeserializer, allModules) + val problemModuleIdWithVersion = allModules.getValue(problemModuleId).moduleIdWithVersion + + // cause: + append("Module \"$problemModuleId\" has a reference to symbol ${idSignature.render()}.") + append(" Neither the module itself nor its dependencies contain such declaration.") + + // explanation: + append("\n\nThis could happen if the required dependency is missing in the project.") + append(" Or if there is a dependency of \"$problemModuleId\" that has a different version in the project") + append(" than the version that \"$problemModuleIdWithVersion\" was initially compiled with.") + + // action items: + append(" Please check that the project configuration is correct and has consistent versions of all required dependencies.") + + // potentially conflicting modules: + appendPotentiallyConflictingDependencies( + header = "The list of \"$problemModuleIdWithVersion\" dependencies that may lead to conflicts:", + allModules = allModules, + potentiallyConflictingDependencies = findPotentiallyConflictingOutgoingDependencies( + problemModuleId = problemModuleId, + allModules = allModules, + ), + moduleIdComparator = userVisibleIrModulesSupport.moduleIdComparator + ) + + // the tree of dependencies: + appendProjectDependencies( + allModules = allModules, + problemModuleIds = setOf(problemModuleId), + problemCause = "This module requires symbol ${idSignature.render()}", + moduleIdComparator = userVisibleIrModulesSupport.moduleIdComparator + ) + } +} + +class NoDeserializerForModule(moduleName: Name, idSignature: IdSignature?) : KotlinIrLinkerIssue() { + override val message = buildString { + append("Could not load module ${moduleName.asString()}") + if (idSignature != null) append(" in an attempt to find deserializer for symbol ${idSignature.render()}.") + } +} + +class SymbolTypeMismatch( + cause: IrSymbolTypeMismatchException, + allModuleDeserializers: Collection, + userVisibleIrModulesSupport: UserVisibleIrModulesSupport +) : KotlinIrLinkerIssue() { + override val message: String = buildString { + val allModules = userVisibleIrModulesSupport.getUserVisibleModules(allModuleDeserializers) + + val idSignature = cause.actual.signature + // There might be multiple declaring modules. Which is also an error, and should re reported separately. + val declaringModuleIds: Set = if (idSignature != null) { + allModuleDeserializers.mapNotNullTo(mutableSetOf()) { deserializer -> + if (idSignature in deserializer) userVisibleIrModulesSupport.getProblemModuleId(deserializer, allModules) else null + } + } else emptySet() + + // cause: + append(cause.message) + + // explanation: + append("\n\nThis could happen if there are two libraries, where one library was compiled against the different version") + append(" of the other library than the one currently used in the project.") + + // action items: + append(" Please check that the project configuration is correct and has consistent versions of dependencies.") + + // potentially conflicting modules: + declaringModuleIds.forEach { declaringModuleId -> + appendPotentiallyConflictingDependencies( + header = "The list of libraries that depend on \"$declaringModuleId\" and may lead to conflicts:", + allModules = allModules, + potentiallyConflictingDependencies = findPotentiallyConflictingIncomingDependencies( + problemModuleId = declaringModuleId, + allModules = allModules, + ), + moduleIdComparator = userVisibleIrModulesSupport.moduleIdComparator + ) + } + + // the tree of dependencies: + appendProjectDependencies( + allModules = allModules, + problemModuleIds = declaringModuleIds, + problemCause = "This module contains ${ + idSignature?.render()?.let { "symbol $it" } ?: "a symbol" + } that is the cause of the conflict", + moduleIdComparator = userVisibleIrModulesSupport.moduleIdComparator + ) + } +} + +private fun UserVisibleIrModulesSupport.getProblemModuleId( + problemModuleDeserializer: IrModuleDeserializer, + allModules: Map +): ResolvedDependencyId = allModules.findMatchingModule(getUserVisibleModuleId(problemModuleDeserializer)).id + +/** + * Do the best effort to find a module that matches the given [moduleId]: + * - If there is a node in the map with such [ResolvedDependencyId] as [moduleId], then return the value from this node. + * - If not, then try to find a [ResolvedDependency] which contains all unique names from the given [moduleId]. This makes sense + * for such cases when the map contains a node for "org.jetbrains.kotlinx:kotlinx-coroutines-core (org.jetbrains.kotlinx:kotlinx-coroutines-core-macosx64)" + * but we are looking just for "org.jetbrains.kotlinx:kotlinx-coroutines-core". + */ +private fun Map.findMatchingModule(moduleId: ResolvedDependencyId): ResolvedDependency { + this[moduleId]?.let { module -> + // Yes, there is a module with such ID. Just return it. + return module + } + + // No, there is no module with such ID. => Find a module with a wider set of unique names. + // Typical case is when we are looking for "org.jetbrains.kotlinx:kotlinx-coroutines-core", but there is no such module. + // But there is "org.jetbrains.kotlinx:kotlinx-coroutines-core (org.jetbrains.kotlinx:kotlinx-coroutines-core-macosx64)" + // that should be used instead. + return values.first { moduleId in it.id } +} + +private fun StringBuilder.appendProjectDependencies( + allModules: Map, + problemModuleIds: Set, + problemCause: String, + moduleIdComparator: Comparator +) { + append("\n\nProject dependencies:") + if (allModules.isEmpty()) { + append(" ") + return + } + + val incomingDependencyIdToDependencies: MutableMap> = mutableMapOf() + allModules.values.forEach { module -> + module.requestedVersionsByIncomingDependencies.keys.forEach { incomingDependencyId -> + incomingDependencyIdToDependencies.getOrPut(incomingDependencyId) { mutableListOf() } += module + } + } + + val renderedModules: MutableSet = mutableSetOf() + var everDependenciesOmitted = false + + fun renderModules(modules: Collection, parentData: Data?) { + val filteredModules: Collection = if (parentData == null) + modules.filter { it.visibleAsFirstLevelDependency } + else + modules + + val sortedModules: List = filteredModules.sortedWith { a, b -> moduleIdComparator.compare(a.id, b.id) } + + sortedModules.forEachIndexed { index, module -> + val data = Data( + parent = parentData, + incomingDependencyId = module.id, // For children. + isLast = index + 1 == sortedModules.size + ) + + append('\n').append(data.regularLinePrefix) + append(module.id) + + val incomingDependencyId: ResolvedDependencyId = parentData?.incomingDependencyId + ?: ResolvedDependencyId.SOURCE_CODE_MODULE_ID + val requestedVersion: ResolvedDependencyVersion = module.requestedVersionsByIncomingDependencies.getValue(incomingDependencyId) + if (!requestedVersion.isEmpty() || !module.selectedVersion.isEmpty()) { + append(": ") + append(requestedVersion.version.ifEmpty { UNKNOWN_VERSION }) + if (requestedVersion != module.selectedVersion) { + append(" -> ") + append(module.selectedVersion.version.ifEmpty { UNKNOWN_VERSION }) + } + } + + val renderedFirstTime = renderedModules.add(module.id) + val dependencies: Collection? = incomingDependencyIdToDependencies[module.id] + + val needToRenderDependencies = when { + renderedFirstTime -> { + // Rendered for the first time => also render dependencies, if any. + true + } + !dependencies.isNullOrEmpty() -> { + // Already rendered at least once. Do not render dependencies, but add a mark that dependencies are omitted. + everDependenciesOmitted = true + append(" (*)") + false + } + else -> { + // Already rendered at least once. No dependencies. + false + } + } + + + if (module.id in problemModuleIds) { + append('\n').append(data.errorLinePrefix) + append("^^^ $problemCause.") + } + + if (needToRenderDependencies && !dependencies.isNullOrEmpty()) { + renderModules(dependencies, data) + } + } + } + + // Find first-level dependencies. I.e. the modules that the source code module directly depends on. + val firstLevelDependencies: Collection = + incomingDependencyIdToDependencies.getValue(ResolvedDependencyId.SOURCE_CODE_MODULE_ID) + + renderModules(firstLevelDependencies, parentData = null) + + if (everDependenciesOmitted) { + append("\n\n(*) - dependencies omitted (listed previously)") + } +} + +private const val UNKNOWN_VERSION = "unknown" + +private class Data(val parent: Data?, val incomingDependencyId: ResolvedDependencyId, val isLast: Boolean) { + val regularLinePrefix: String + get() { + return generateSequence(this) { it.parent }.map { + if (it === this) { + if (it.isLast) "\u2514\u2500\u2500\u2500 " /* └─── */ else "\u251C\u2500\u2500\u2500 " /* ├─── */ + } else { + if (it.isLast) " " else "\u2502 " /* │ */ + } + }.toList().asReversed().joinToString(separator = "") + } + + val errorLinePrefix: String + get() { + return generateSequence(this) { it.parent }.map { + if (it.isLast) " " else "\u2502 " /* │ */ + }.toList().asReversed().joinToString(separator = "") + } +} + +/** + * Find all outgoing dependencies of [problemModuleId] that might conflict with [problemModuleId] because they have + * different (overridden) selected version then the version that [problemModuleId] was initially compiled with. + */ +private fun findPotentiallyConflictingOutgoingDependencies( + problemModuleId: ResolvedDependencyId, + allModules: Map +): Map { + data class OutgoingDependency( + val id: ResolvedDependencyId, + val requestedVersion: ResolvedDependencyVersion, + val selectedVersion: ResolvedDependencyVersion + ) + + // Reverse dependency index. + val outgoingDependenciesIndex: MutableMap> = mutableMapOf() + + allModules.values.forEach { module -> + module.requestedVersionsByIncomingDependencies.forEach { (incomingDependencyId, requestedVersion) -> + outgoingDependenciesIndex.getOrPut(incomingDependencyId) { mutableListOf() } += OutgoingDependency( + id = module.id, + requestedVersion = requestedVersion, + selectedVersion = module.selectedVersion + ) + } + } + + val dependencyStatesMap: MutableMap> = mutableMapOf() + + fun recurse(moduleId: ResolvedDependencyId, underConflictingDependency: Boolean) { + val outgoingDependencies: List = outgoingDependenciesIndex[moduleId].orEmpty() + + outgoingDependencies.forEach { outgoingDependency -> + val dependencyState: DependencyState = when { + underConflictingDependency -> { + // Can't guarantee that any library that is a dependency of a potentially conflicting dependency + // is not a potentially conflicting dependency itself. + DependencyState( + conflictReason = PotentialConflictReason( + kind = BEHIND_CONFLICTING_DEPENDENCY, + conflictingModuleId = outgoingDependency.id + ) + ) + } + outgoingDependency.selectedVersion.isEmpty() -> { + // Selected version is empty (unknown). Can't guarantee that this is the same library that was requested even if + // the requested version is also empty (unknown). + DependencyState( + conflictReason = PotentialConflictReason( + kind = UNKNOWN_SELECTED_VERSION, + conflictingModuleId = outgoingDependency.id, + requestedVersion = outgoingDependency.requestedVersion + ) + ) + } + outgoingDependency.requestedVersion != outgoingDependency.selectedVersion -> { + // The requested and selected versions don't match. + DependencyState( + conflictReason = PotentialConflictReason( + kind = REQUESTED_SELECTED_VERSIONS_MISMATCH, + conflictingModuleId = outgoingDependency.id, + requestedVersion = outgoingDependency.requestedVersion, + selectedVersion = outgoingDependency.selectedVersion + ) + ) + } + else -> DependencyState.SUCCESS + } + + val dependencyStates: MutableSet = dependencyStatesMap.getOrPut(outgoingDependency.id) { mutableSetOf() } + val notBeenHereYet = dependencyStates.add(dependencyState) + + if (notBeenHereYet) { + // Don't visit the same dependency twice. + recurse(moduleId = outgoingDependency.id, underConflictingDependency = dependencyState.conflictReason != null) + } + } + } + + recurse(moduleId = problemModuleId, underConflictingDependency = false) + + return dependencyStatesMap.describeDependencyStates { potentialConflictReason -> + when (potentialConflictReason.kind) { + UNKNOWN_SELECTED_VERSION -> { + "a library with unknown version" + } + REQUESTED_SELECTED_VERSIONS_MISMATCH -> { + val requested = potentialConflictReason.conflictingModuleId.withVersion(potentialConflictReason.requestedVersion) + "was initially compiled with \"$requested\"" + } + BEHIND_CONFLICTING_DEPENDENCY -> { + "a dependency of the library with unknown version or versions mismatch: \"${potentialConflictReason.conflictingModuleId}\"" + } + } + } +} + +/** + * Find all incoming dependencies of [problemModuleId] that might conflict with [problemModuleId] because they were + * initially compiled with the different version of [problemModuleId] than the one used in the project. + */ +private fun findPotentiallyConflictingIncomingDependencies( + problemModuleId: ResolvedDependencyId, + allModules: Map +): Map { + + val dependencyStatesMap: MutableMap> = mutableMapOf() + + fun recurse(moduleId: ResolvedDependencyId, aboveConflictingDependency: Boolean) { + val module = allModules.findMatchingModule(moduleId) + + module.requestedVersionsByIncomingDependencies.forEach { (incomingDependencyId, requestedVersion) -> + if (incomingDependencyId == ResolvedDependencyId.SOURCE_CODE_MODULE_ID) return@forEach + + val dependencyState: DependencyState = when { + aboveConflictingDependency -> { + // Can't guarantee that any library that is an incoming dependency of a potentially conflicting dependency + // is not a potentially conflicting dependency itself. + DependencyState( + conflictReason = PotentialConflictReason( + kind = BEHIND_CONFLICTING_DEPENDENCY, + conflictingModuleId = module.id + ) + ) + } + module.selectedVersion.isEmpty() -> { + // Selected version is empty (unknown). Can't guarantee that this is the same library that was requested even if + // the requested version is also empty (unknown). + DependencyState( + conflictReason = PotentialConflictReason( + kind = UNKNOWN_SELECTED_VERSION, + conflictingModuleId = module.id, + requestedVersion = requestedVersion + ) + ) + } + requestedVersion != module.selectedVersion -> { + // The requested and selected versions don't match. + DependencyState( + conflictReason = PotentialConflictReason( + kind = REQUESTED_SELECTED_VERSIONS_MISMATCH, + conflictingModuleId = module.id, + requestedVersion = requestedVersion, + selectedVersion = module.selectedVersion + ) + ) + } + else -> DependencyState.SUCCESS + } + + val dependencyStates: MutableSet = dependencyStatesMap.getOrPut(incomingDependencyId) { mutableSetOf() } + val notBeenHereYet = dependencyStates.add(dependencyState) + + if (notBeenHereYet) { + // Don't visit the same dependency twice. + recurse(moduleId = incomingDependencyId, aboveConflictingDependency = dependencyState.isConflicting) + } + } + } + + recurse(moduleId = problemModuleId, aboveConflictingDependency = false) + + return dependencyStatesMap.describeDependencyStates { potentialConflictReason -> + when (potentialConflictReason.kind) { + UNKNOWN_SELECTED_VERSION -> { + "depends on the library with unknown version: \"${potentialConflictReason.conflictingModuleId}\"" + } + REQUESTED_SELECTED_VERSIONS_MISMATCH -> { + val requested = potentialConflictReason.conflictingModuleId.withVersion(potentialConflictReason.requestedVersion) + val selected = potentialConflictReason.conflictingModuleId.withVersion(potentialConflictReason.selectedVersion) + "was compiled against \"$requested\" but \"$selected\" is used in the project" + } + BEHIND_CONFLICTING_DEPENDENCY -> { + "depends on the library with unknown version or versions mismatch: \"${potentialConflictReason.conflictingModuleId}\"" + } + } + } +} + +private typealias PotentialConflictDescription = String + +private class PotentialConflictReason( + val kind: PotentialConflictKind, + val conflictingModuleId: ResolvedDependencyId, + val requestedVersion: ResolvedDependencyVersion = ResolvedDependencyVersion.EMPTY, + val selectedVersion: ResolvedDependencyVersion = ResolvedDependencyVersion.EMPTY +) { + override fun equals(other: Any?) = + other is PotentialConflictReason && other.kind == kind && other.conflictingModuleId == conflictingModuleId + + override fun hashCode() = kind.hashCode() + 31 * conflictingModuleId.hashCode() + + companion object { + val Collection.mostSignificantConflictReasons: Collection + get() { + val mapping: Map> = groupBy { it.kind } + val mostSignificantConflictKind: PotentialConflictKind = mapping.keys.mostSignificantConflictKind ?: return emptyList() + return mapping.getValue(mostSignificantConflictKind) + } + } +} + + +// Important: Enum entries must be declared in the order of most-to-least significant reasons. +private enum class PotentialConflictKind { + UNKNOWN_SELECTED_VERSION, + REQUESTED_SELECTED_VERSIONS_MISMATCH, + BEHIND_CONFLICTING_DEPENDENCY; + + companion object { + val Collection.mostSignificantConflictKind: PotentialConflictKind? + get() = minOrNull() + } +} + +private data class DependencyState(val conflictReason: PotentialConflictReason? /* Null assumes success. */) { + val isConflicting: Boolean get() = conflictReason != null + + companion object { + val SUCCESS = DependencyState(conflictReason = null) + } +} + +private fun Map>.describeDependencyStates( + getDescription: (PotentialConflictReason) -> PotentialConflictDescription +): Map = mapNotNull { (dependencyId, dependencyStates) -> + val mostSignificantConflictReasons = dependencyStates.mapNotNull { it.conflictReason }.mostSignificantConflictReasons + + when { + mostSignificantConflictReasons.isEmpty() -> { + // No conflicts. + return@mapNotNull null + } + mostSignificantConflictReasons.first().kind == BEHIND_CONFLICTING_DEPENDENCY -> { + // Ignore dependency of conflicting dependency if this library has no conflicts when referred from + // non-conflicting dependencies. + val hasNoConflictsBehindNonConflictingDependencies = dependencyStates.any { !it.isConflicting } + if (hasNoConflictsBehindNonConflictingDependencies) return@mapNotNull null + } + else -> Unit + } + + dependencyId to mostSignificantConflictReasons.joinToString(separator = "; ") { getDescription(it) } +}.toMap() + +private fun StringBuilder.appendPotentiallyConflictingDependencies( + header: String, + allModules: Map, + potentiallyConflictingDependencies: Map, + moduleIdComparator: Comparator +) { + if (potentiallyConflictingDependencies.isEmpty()) return + + append("\n\n$header") + + val paddingSize = (potentiallyConflictingDependencies.size + 1).toString().length + potentiallyConflictingDependencies.toSortedMap(moduleIdComparator).entries.forEachIndexed { index, (moduleId, potentialConflictReason) -> + val padding = (index + 1).toString().padStart(paddingSize, ' ') + val moduleIdWithVersion = allModules.getValue(moduleId).moduleIdWithVersion + append("\n$padding. \"$moduleIdWithVersion\" ($potentialConflictReason)") + } +} diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/UserVisibleIrModulesSupport.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/UserVisibleIrModulesSupport.kt new file mode 100644 index 00000000000..4eae3aa3c1d --- /dev/null +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/linkerissues/UserVisibleIrModulesSupport.kt @@ -0,0 +1,229 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.common.serialization.linkerissues + +import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializer +import org.jetbrains.kotlin.descriptors.konan.DeserializedKlibModuleOrigin +import org.jetbrains.kotlin.descriptors.konan.KlibModuleOrigin +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.uniqueName +import org.jetbrains.kotlin.utils.* + +open class UserVisibleIrModulesSupport(protected val externalDependenciesLoader: ExternalDependenciesLoader) { + /** + * Load external [ResolvedDependency]s provided by the build system. These dependencies: + * - all have [ResolvedDependency.selectedVersion] specified + * - keep the information about which modules are first-level dependencies (i.e. the modules that the source code module + * directly depends on) and indirect (transitive) dependencies + * - miss modules provided by Kotlin/Native distribution (stdlib, endorsed and platform libraries), as they are + * not visible to the build system + */ + interface ExternalDependenciesLoader { + fun load(): Collection + + companion object { + val EMPTY = object : ExternalDependenciesLoader { + override fun load(): Collection = emptyList() + } + + fun from(externalDependenciesFile: File?, onMalformedExternalDependencies: (String) -> Unit): ExternalDependenciesLoader = + if (externalDependenciesFile != null) + object : ExternalDependenciesLoader { + override fun load(): Collection { + return if (externalDependenciesFile.exists) { + // Deserialize external dependencies from the [externalDependenciesFile]. + val externalDependenciesText = String(externalDependenciesFile.readBytes()) + ResolvedDependenciesSupport.deserialize(externalDependenciesText) { lineNo, line -> + onMalformedExternalDependencies("Malformed external dependencies at $externalDependenciesFile:$lineNo: $line") + } + } else emptyList() + } + } + else + EMPTY + } + } + + fun getUserVisibleModuleId(deserializer: IrModuleDeserializer): ResolvedDependencyId { + val nameFromMetadataModuleHeader: String = deserializer.moduleFragment.name.asStringStripSpecialMarkers() + val nameFromKlibManifest: String? = deserializer.kotlinLibrary?.uniqueName + + return ResolvedDependencyId(listOfNotNull(nameFromMetadataModuleHeader, nameFromKlibManifest)) + } + + open fun getUserVisibleModules(deserializers: Collection): Map { + return mergedModules(deserializers) + } + + /** + * Load [ResolvedDependency]s that represent all libraries participating in the compilation. Includes external dependencies, + * but without version and hierarchy information. Also includes the libraries that are not visible to the build system + * (and therefore are missing in [ExternalDependenciesLoader.load]) but are provided by the compiler. For Kotlin/Native such + * libraries are stdlib, endorsed and platform libraries. + */ + protected open fun modulesFromDeserializers(deserializers: Collection): Map { + val modules: Map = deserializers.associate { deserializer -> + val moduleId = getUserVisibleModuleId(deserializer) + val module = ResolvedDependency( + id = moduleId, + // TODO: support extracting all the necessary details for non-Native libs: selectedVersion, requestedVersions, artifacts + selectedVersion = ResolvedDependencyVersion.EMPTY, + // Assumption: As we don't know for sure which modules the source code module depends on directly and which modules + // it depends on transitively, so let's assume it depends on all modules directly. + requestedVersionsByIncomingDependencies = mutableMapOf(ResolvedDependencyId.SOURCE_CODE_MODULE_ID to ResolvedDependencyVersion.EMPTY), + artifactPaths = mutableSetOf() + ) + + val outgoingDependencyIds = deserializer.moduleDependencies.map { getUserVisibleModuleId(it) } + + moduleId to ModuleWithUninitializedDependencies(module, outgoingDependencyIds) + } + + // Stamp dependencies. + return modules.stampDependenciesWithRequestedVersionEqualToSelectedVersion() + } + + /** + * The result of the merge of [ExternalDependenciesLoader.load] and [modulesFromDeserializers]. + */ + protected fun mergedModules(deserializers: Collection): MutableMap { + // First, load external dependencies. + val externalDependencyModules: Collection = externalDependenciesLoader.load() + + val externalDependencyModulesByNames: Map = + mutableMapOf().apply { + externalDependencyModules.forEach { externalDependency -> + externalDependency.id.uniqueNames.forEach { uniqueName -> + this[uniqueName] = externalDependency + } + } + } + + fun findMatchingExternalDependencyModule(moduleId: ResolvedDependencyId): ResolvedDependency? = + moduleId.uniqueNames.firstNotNullOfOrNull { uniqueName -> externalDependencyModulesByNames[uniqueName] } + + // The build system may express a group of modules where one module is a library KLIB and one or more modules + // are just C-interop KLIBs as a single module with multiple artifacts. We need to expand them so that every particular + // module/artifact will be represented as an individual [ResolvedDependency] instance. + val artifactPathsToOriginModules: MutableMap = mutableMapOf() + externalDependencyModules.forEach { originModule -> + originModule.artifactPaths.forEach { artifactPath -> artifactPathsToOriginModules[artifactPath] = originModule } + } + + // Collect "provided" modules, i.e. modules that are missing in "external dependencies". + val providedModules = mutableListOf() + + // Next, merge external dependencies with dependencies from deserializers. + modulesFromDeserializers(deserializers).forEach { (moduleId, module) -> + val externalDependencyModule = findMatchingExternalDependencyModule(moduleId) + if (externalDependencyModule != null) { + // Just add missing dependencies to the same module in [mergedModules]. + module.requestedVersionsByIncomingDependencies.forEach { (incomingDependencyId, requestedVersion) -> + val adjustedIncomingDependencyId = findMatchingExternalDependencyModule(incomingDependencyId)?.id + ?: incomingDependencyId + if (adjustedIncomingDependencyId !in externalDependencyModule.requestedVersionsByIncomingDependencies) { + externalDependencyModule.requestedVersionsByIncomingDependencies[adjustedIncomingDependencyId] = requestedVersion + } + } + } else { + val originModuleVersion = module.artifactPaths.firstNotNullOfOrNull { artifactPathsToOriginModules[it] }?.selectedVersion + if (originModuleVersion != null) { + // Handle artifacts that needs to be represented as individual [ResolvedDependency] objects. + module.selectedVersion = originModuleVersion + + val incomingDependencyIdsToStampRequestedVersion = module.requestedVersionsByIncomingDependencies + .mapNotNull { (incomingDependencyId, requestedVersion) -> + if (requestedVersion.isEmpty()) incomingDependencyId else null + } + incomingDependencyIdsToStampRequestedVersion.forEach { incomingDependencyId -> + module.requestedVersionsByIncomingDependencies[incomingDependencyId] = originModuleVersion + } + } else { + // Just keep the module as is. If it has no incoming dependencies, then treat it as + // the first-level dependency module (i.e. the module that only the source code module depends on). + if (module.requestedVersionsByIncomingDependencies.isEmpty()) { + module.requestedVersionsByIncomingDependencies[ResolvedDependencyId.SOURCE_CODE_MODULE_ID] = module.selectedVersion + } + } + + // Patch incoming dependencies. + module.requestedVersionsByIncomingDependencies.mapNotNull { (incomingDependencyId, requestedVersion) -> + val adjustedIncomingDependencyId = findMatchingExternalDependencyModule(incomingDependencyId)?.id + ?: return@mapNotNull null + Triple(incomingDependencyId, adjustedIncomingDependencyId, requestedVersion) + }.forEach { (incomingDependencyId, adjustedIncomingDependencyId, requestedVersion) -> + module.requestedVersionsByIncomingDependencies.remove(incomingDependencyId) + module.requestedVersionsByIncomingDependencies[adjustedIncomingDependencyId] = requestedVersion + } + + providedModules += module + } + } + + return (externalDependencyModules + providedModules).associateByTo(mutableMapOf()) { it.id } + } + + protected data class ModuleWithUninitializedDependencies( + val module: ResolvedDependency, + val outgoingDependencyIds: List + ) + + private fun Map.stampDependenciesWithRequestedVersionEqualToSelectedVersion(): Map { + return mapValues { (moduleId, moduleWithUninitializedDependencies) -> + val (module, outgoingDependencyIds) = moduleWithUninitializedDependencies + outgoingDependencyIds.forEach { outgoingDependencyId -> + val dependencyModule = getValue(outgoingDependencyId).module + dependencyModule.requestedVersionsByIncomingDependencies[moduleId] = dependencyModule.selectedVersion + } + module + } + } + + // TODO: find a way to access KotlinLibrary without descriptors + protected val IrModuleDeserializer.kotlinLibrary: KotlinLibrary? + get() = (moduleDescriptor.getCapability(KlibModuleOrigin.CAPABILITY) as? DeserializedKlibModuleOrigin)?.library + + val moduleIdComparator: Comparator = Comparator { a, b -> + when { + a == b -> 0 + // Kotlin libs go lower. + a.isKotlinLibrary && !b.isKotlinLibrary -> 1 + !a.isKotlinLibrary && b.isKotlinLibrary -> -1 + // Modules with simple names go upper as they are most likely user-made libs. + a.hasSimpleName && !b.hasSimpleName -> -1 + !a.hasSimpleName && b.hasSimpleName -> 1 + // Else: just compare by names. + else -> { + val aUniqueNames = a.uniqueNames.iterator() + val bUniqueNames = b.uniqueNames.iterator() + + while (aUniqueNames.hasNext() && bUniqueNames.hasNext()) { + val diff = aUniqueNames.next().compareTo(bUniqueNames.next()) + if (diff != 0) return@Comparator diff + } + + when { + aUniqueNames.hasNext() -> 1 + bUniqueNames.hasNext() -> -1 + else -> 0 + } + } + } + } + + protected open val ResolvedDependencyId.isKotlinLibrary: Boolean + get() = uniqueNames.any { uniqueName -> uniqueName.startsWith(KOTLIN_LIBRARY_PREFIX) } + + protected open val ResolvedDependencyId.hasSimpleName: Boolean + get() = uniqueNames.all { uniqueName -> uniqueName.none { it == '.' || it == ':' } } + + companion object { + const val KOTLIN_LIBRARY_PREFIX = "org.jetbrains.kotlin" + + val DEFAULT = UserVisibleIrModulesSupport(ExternalDependenciesLoader.EMPTY) + } +} diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt index 9457cb48971..8a03f988ef5 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcFileDeserializer.kt @@ -8,7 +8,6 @@ package org.jetbrains.kotlin.ir.backend.js.ic import org.jetbrains.kotlin.backend.common.overrides.DefaultFakeOverrideClassFilter import org.jetbrains.kotlin.backend.common.serialization.* import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData -import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.ir.backend.js.JsMappingState import org.jetbrains.kotlin.ir.backend.js.JsStatementOrigins import org.jetbrains.kotlin.ir.backend.js.lower.serialization.ir.JsIrLinker @@ -36,7 +35,6 @@ class IcFileDeserializer( allowErrorNodes: Boolean, deserializeInlineFunctions: Boolean, val moduleDeserializer: IrModuleDeserializer, - val handleNoModuleDeserializerFound: (IdSignature, ModuleDescriptor, Collection) -> IrModuleDeserializer, val originalEnqueue: IdSignature.(IcFileDeserializer) -> Unit, val icFileData: SerializedIcDataForFile, val mappingState: JsMappingState, @@ -82,9 +80,8 @@ class IcFileDeserializer( // TODO: reference lowered declarations cross-module if (kind == BinarySymbolData.SymbolKind.FILE_SYMBOL) return file.symbol - val actualModuleDeserializer = - moduleDeserializer.findModuleDeserializerForTopLevelId(idSig) - ?: handleNoModuleDeserializerFound(idSig, moduleDeserializer.moduleDescriptor, moduleDeserializer.moduleDependencies) + val actualModuleDeserializer = moduleDeserializer.findModuleDeserializerForTopLevelId(idSig) + ?: linker.handleSignatureIdNotFoundInModuleWithDependencies(idSig, moduleDeserializer) return actualModuleDeserializer.deserializeIrSymbol(idSig, kind) } diff --git a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt index 069f3bef74b..86fc9e053dd 100644 --- a/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt +++ b/compiler/ir/serialization.js/src/org/jetbrains/kotlin/ir/backend/js/ic/IcModuleDeserializer.kt @@ -141,7 +141,6 @@ class IcModuleDeserializer( allowErrorNodes, strategy.inlineBodies, moduleDeserializer, - linker::handleNoModuleDeserializerFound, { fileDeserializer -> originalEnqueue(fileDeserializer) }, icFileData, mapping.state, diff --git a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt index 3934dc4b5b3..71a801d4481 100644 --- a/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt +++ b/compiler/util-io/src/org/jetbrains/kotlin/konan/file/File.kt @@ -58,6 +58,7 @@ data class File constructor(internal val javaPath: Path) { get() = if (exists) listFiles else emptyList() fun child(name: String) = File(this, name) + fun startsWith(another: File) = javaPath.startsWith(another.javaPath) fun copyTo(destination: File) { Files.copy(javaPath, destination.javaPath, StandardCopyOption.REPLACE_EXISTING) diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt index c24e5144c29..1bad26522c7 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt @@ -6,6 +6,8 @@ package org.jetbrains.kotlin.backend.konan import com.intellij.openapi.project.Project +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.UserVisibleIrModulesSupport +import org.jetbrains.kotlin.backend.konan.serialization.KonanUserVisibleIrModulesSupport import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity @@ -91,6 +93,15 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration val resolvedLibraries get() = resolve.resolvedLibraries + internal val userVisibleIrModulesSupport = KonanUserVisibleIrModulesSupport( + externalDependenciesLoader = UserVisibleIrModulesSupport.ExternalDependenciesLoader.from( + externalDependenciesFile = configuration.get(KonanConfigKeys.EXTERNAL_DEPENDENCIES)?.let(::File), + onMalformedExternalDependencies = { warningMessage -> + configuration.report(CompilerMessageSeverity.STRONG_WARNING, warningMessage) + }), + konanKlibDir = File(distribution.klib) + ) + internal val cacheSupport = CacheSupport(configuration, resolvedLibraries, target, produce) internal val cachedLibraries: CachedLibraries @@ -206,10 +217,10 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration File(distribution.defaultNatives(target)).child("exceptionsSupport.bc").absolutePath internal val nativeLibraries: List = - configuration.getList(KonanConfigKeys.NATIVE_LIBRARY_FILES) + configuration.getList(KonanConfigKeys.NATIVE_LIBRARY_FILES) internal val includeBinaries: List = - configuration.getList(KonanConfigKeys.INCLUDED_BINARY_FILES) + configuration.getList(KonanConfigKeys.INCLUDED_BINARY_FILES) internal val languageVersionSettings = configuration.get(CommonConfigurationKeys.LANGUAGE_VERSION_SETTINGS)!! diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt index 07090616be4..089cd055036 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/PsiToIr.kt @@ -101,7 +101,8 @@ internal fun Context.psiToIr( stubGenerator, irProviderForCEnumsAndCStructs, exportedDependencies, - config.cachedLibraries + config.cachedLibraries, + config.userVisibleIrModulesSupport ).also { linker -> // context.config.librariesWithDependencies could change at each iteration. diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt index 2c494386551..9d6fa239641 100644 --- a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanIrlinker.kt @@ -20,6 +20,7 @@ import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideBuilder import org.jetbrains.kotlin.backend.common.overrides.FakeOverrideClassFilter import org.jetbrains.kotlin.backend.common.serialization.* import org.jetbrains.kotlin.backend.common.serialization.encodings.BinarySymbolData +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.UserVisibleIrModulesSupport import org.jetbrains.kotlin.backend.konan.CachedLibraries import org.jetbrains.kotlin.backend.konan.descriptors.isInteropLibrary import org.jetbrains.kotlin.backend.konan.ir.interop.IrProviderForCEnumAndCStructStubs @@ -72,7 +73,8 @@ internal class KonanIrLinker( private val stubGenerator: DeclarationStubGenerator, private val cenumsProvider: IrProviderForCEnumAndCStructStubs, exportedDependencies: List, - private val cachedLibraries: CachedLibraries + private val cachedLibraries: CachedLibraries, + override val userVisibleIrModulesSupport: UserVisibleIrModulesSupport ) : KotlinIrLinker(currentModule, messageLogger, builtIns, symbolTable, exportedDependencies) { companion object { @@ -204,7 +206,7 @@ internal class KonanIrLinker( val descriptor = resolveDescriptor(idSig) val actualModule = descriptor.module if (actualModule !== moduleDescriptor) { - val moduleDeserializer = deserializersForModules[actualModule.name.asString()] ?: error("No module deserializer for $actualModule") + val moduleDeserializer = resolveModuleDeserializer(actualModule, idSig) moduleDeserializer.addModuleReachableTopLevel(idSig) return symbolTable.referenceClassFromLinker(idSig) } diff --git a/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanUserVisibleIrModulesSupport.kt b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanUserVisibleIrModulesSupport.kt new file mode 100644 index 00000000000..8bca23e3470 --- /dev/null +++ b/kotlin-native/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/serialization/KonanUserVisibleIrModulesSupport.kt @@ -0,0 +1,152 @@ +/* + * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.backend.konan.serialization + +import org.jetbrains.kotlin.backend.common.serialization.IrModuleDeserializer +import org.jetbrains.kotlin.backend.common.serialization.linkerissues.UserVisibleIrModulesSupport +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.konan.library.KONAN_PLATFORM_LIBS_NAME_PREFIX +import org.jetbrains.kotlin.konan.library.KONAN_STDLIB_NAME +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.RequiredUnresolvedLibrary +import org.jetbrains.kotlin.library.unresolvedDependencies +import org.jetbrains.kotlin.utils.ResolvedDependency +import org.jetbrains.kotlin.utils.ResolvedDependencyArtifactPath +import org.jetbrains.kotlin.utils.ResolvedDependencyId +import org.jetbrains.kotlin.utils.ResolvedDependencyVersion + +class KonanUserVisibleIrModulesSupport( + externalDependenciesLoader: ExternalDependenciesLoader, + private val konanKlibDir: File +) : UserVisibleIrModulesSupport(externalDependenciesLoader) { + override fun getUserVisibleModules(deserializers: Collection): Map { + return compressedModules(deserializers) + } + + override fun modulesFromDeserializers(deserializers: Collection): Map { + val moduleToCompilerVersion: MutableMap = mutableMapOf() + val kotlinNativeBundledLibraries: MutableSet = mutableSetOf() + + // Transform deserializers to [ModuleWithUninitializedDependencies]s. + val modules: Map = deserializers.mapNotNull { deserializer -> + val library: KotlinLibrary = deserializer.kotlinLibrary ?: return@mapNotNull null + + val compilerVersion: ResolvedDependencyVersion = library.compilerVersion + val isDefaultLibrary = library.isDefaultLibrary + + // For default libraries the version is the same as the version of the compiler. + // Note: Empty string means missing (unknown) version. + val libraryVersion: ResolvedDependencyVersion = if (isDefaultLibrary) compilerVersion else ResolvedDependencyVersion.EMPTY + + val moduleId = getUserVisibleModuleId(deserializer) + val module = ResolvedDependency( + id = moduleId, + selectedVersion = libraryVersion, + requestedVersionsByIncomingDependencies = mutableMapOf(), // To be initialized in a separate pass below. + artifactPaths = mutableSetOf(ResolvedDependencyArtifactPath(library.libraryFile.absolutePath)) + ) + + moduleToCompilerVersion[moduleId] = compilerVersion + if (isDefaultLibrary) kotlinNativeBundledLibraries += moduleId + + // Don't rely on dependencies in IrModuleDeserializer. In Kotlin/Native each module depends on all other modules, + // and this contradicts with the real module dependencies as written in KLIB manifest files. + val outgoingDependencyIds = library.unresolvedDependencies.map { it.moduleId } + + moduleId to ModuleWithUninitializedDependencies(module, outgoingDependencyIds) + }.toMap() + + // Stamp dependencies. + return modules.mapValues { (moduleId, moduleWithUninitializedDependencies) -> + val (module, outgoingDependencyIds) = moduleWithUninitializedDependencies + outgoingDependencyIds.forEach { outgoingDependencyId -> + val outgoingDependencyModule = modules.getValue(outgoingDependencyId).module + if (outgoingDependencyId in kotlinNativeBundledLibraries) { + outgoingDependencyModule.requestedVersionsByIncomingDependencies[moduleId] = moduleToCompilerVersion.getValue(moduleId) + } else { + outgoingDependencyModule.requestedVersionsByIncomingDependencies[moduleId] = outgoingDependencyModule.selectedVersion + } + } + module + } + } + + /** + * This is an optimization to avoid displaying 100+ Kotlin/Native platform libraries to the user. + * Instead, lets compress them into a single row and avoid excessive output. + */ + private fun compressedModules(deserializers: Collection): Map { + val compressedModules: MutableMap = mergedModules(deserializers) + + var platformLibrariesVersion: ResolvedDependencyVersion? = null // Must be the same version to succeed. + val platformLibraries: MutableList = mutableListOf() // All platform libraries to be patched. + val outgoingDependencyIds: MutableSet = mutableSetOf() // All outgoing dependencies from platform libraries. + + for ((moduleId, module) in compressedModules) { + if (moduleId.isKonanPlatformLibrary) { + if (ResolvedDependencyId.SOURCE_CODE_MODULE_ID !in module.requestedVersionsByIncomingDependencies) { + continue + } + + platformLibrariesVersion = when (platformLibrariesVersion) { + null, module.selectedVersion -> module.selectedVersion + else -> { + // Multiple versions of platform libs. Give up. + return compressedModules + } + } + + platformLibraries += module + } else { + module.requestedVersionsByIncomingDependencies.keys.forEach { incomingDependencyId -> + if (incomingDependencyId.isKonanPlatformLibrary) { + outgoingDependencyIds += moduleId + } + } + } + } + + if (platformLibraries.isNotEmpty()) { + platformLibraries.forEach { it.visibleAsFirstLevelDependency = false } + + val compressedModuleId = ResolvedDependencyId("$KONAN_PLATFORM_LIBS_NAME_PREFIX* (${platformLibraries.size} libraries)") + val compressedModule = ResolvedDependency( + id = compressedModuleId, + selectedVersion = platformLibrariesVersion!!, + requestedVersionsByIncomingDependencies = mutableMapOf(ResolvedDependencyId.SOURCE_CODE_MODULE_ID to platformLibrariesVersion), + artifactPaths = mutableSetOf() + ) + + outgoingDependencyIds.forEach { outgoingDependencyId -> + val outgoingDependency = compressedModules.getValue(outgoingDependencyId) + outgoingDependency.requestedVersionsByIncomingDependencies[compressedModuleId] = compressedModule.selectedVersion + } + + compressedModules[compressedModuleId] = compressedModule + } + + return compressedModules + } + + private val KotlinLibrary.compilerVersion: ResolvedDependencyVersion + get() = ResolvedDependencyVersion(versions.compilerVersion.orEmpty()) + + // This is much safer check then KotlinLibrary.isDefault, which may return false even for "stdlib" when + // Kotlin/Native compiler is running with "-nostdlib", "-no-endorsed-libs", "-no-default-libs" arguments. + private val KotlinLibrary.isDefaultLibrary: Boolean + get() = libraryFile.startsWith(konanKlibDir) + + override val ResolvedDependencyId.isKotlinLibrary: Boolean + get() = uniqueNames.any { uniqueName -> uniqueName == KONAN_STDLIB_NAME || uniqueName.startsWith(KOTLIN_LIBRARY_PREFIX) } + + companion object { + private val RequiredUnresolvedLibrary.moduleId: ResolvedDependencyId + get() = ResolvedDependencyId(path) // Yep, it's named "path" but in fact holds unique name of the library. + + private val ResolvedDependencyId.isKonanPlatformLibrary: Boolean + get() = uniqueNames.any { it.startsWith(KONAN_PLATFORM_LIBS_NAME_PREFIX) } + } +}