From 5ede65c5255ed0220b0edac8b38d2b424d4acf15 Mon Sep 17 00:00:00 2001 From: Georgy Bronnikov Date: Fri, 29 Nov 2019 11:49:20 +0300 Subject: [PATCH] JVM_IR: read Klib --- .../compiler/TopDownAnalyzerFacadeForJVM.kt | 58 ++++++++++++- .../kotlin/backend/jvm/JvmBackendFacade.kt | 26 +++++- .../backend/jvm/codegen/ClassCodegen.kt | 2 +- .../backend/jvm/lower/FileClassLowering.kt | 27 +++++- .../org/jetbrains/kotlin/ir/util/IrUtils.kt | 36 ++++++++ .../DeserializeDescriptorReference.kt | 4 +- .../common/serialization/KotlinIrLinker.kt | 24 +++++- .../ir/backend/jvm/JvmLibraryResolver.kt | 72 ++++++++++++++++ .../JvmDescriptorReferenceDeserializer.kt | 26 ++++++ .../serialization/JvmDescriptorUniqIdAware.kt | 85 +++++++++++++++++++ .../backend/jvm/serialization/JvmIrLinker.kt | 75 ++++++++++++++++ .../ir/backend/jvm/serialization/Utils.kt | 14 +++ compiler/testData/cli/jvm/extraHelp.out | 1 + .../kotlin/codegen/GenerationUtils.kt | 28 ++++-- .../kotlin/resolve/lazy/JvmResolveUtil.kt | 12 ++- 15 files changed, 462 insertions(+), 28 deletions(-) create mode 100644 compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/JvmLibraryResolver.kt create mode 100644 compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorReferenceDeserializer.kt create mode 100644 compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorUniqIdAware.kt create mode 100644 compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmIrLinker.kt create mode 100644 compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/Utils.kt diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/TopDownAnalyzerFacadeForJVM.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/TopDownAnalyzerFacadeForJVM.kt index 119f3fcb796..f5c7c19e0e9 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/TopDownAnalyzerFacadeForJVM.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/TopDownAnalyzerFacadeForJVM.kt @@ -37,14 +37,20 @@ import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.frontend.java.di.createContainerForLazyResolveWithJava import org.jetbrains.kotlin.frontend.java.di.initJvmBuiltInsForTopDownAnalysis import org.jetbrains.kotlin.frontend.java.di.initialize import org.jetbrains.kotlin.incremental.components.ExpectActualTracker import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.ir.backend.jvm.jvmLibrariesProvidedByDefault import org.jetbrains.kotlin.javac.components.JavacBasedClassFinder import org.jetbrains.kotlin.javac.components.JavacBasedSourceElementFactory import org.jetbrains.kotlin.javac.components.StubJavaResolverCache +import org.jetbrains.kotlin.konan.properties.propertyList +import org.jetbrains.kotlin.konan.util.KlibMetadataFactories +import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS +import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.load.java.JavaClassesTracker import org.jetbrains.kotlin.load.java.lazy.ModuleClassResolver import org.jetbrains.kotlin.load.java.structure.JavaClass @@ -65,6 +71,8 @@ import org.jetbrains.kotlin.resolve.jvm.extensions.PackageFragmentProviderExtens import org.jetbrains.kotlin.resolve.lazy.KotlinCodeAnalyzer import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory +import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer +import org.jetbrains.kotlin.storage.LockBasedStorageManager import org.jetbrains.kotlin.storage.StorageManager import java.util.* import kotlin.reflect.KFunction1 @@ -79,11 +87,12 @@ object TopDownAnalyzerFacadeForJVM { configuration: CompilerConfiguration, packagePartProvider: (GlobalSearchScope) -> PackagePartProvider, declarationProviderFactory: (StorageManager, Collection) -> DeclarationProviderFactory = ::FileBasedDeclarationProviderFactory, - sourceModuleSearchScope: GlobalSearchScope = newModuleSearchScope(project, files) + sourceModuleSearchScope: GlobalSearchScope = newModuleSearchScope(project, files), + klibList: List = emptyList() ): AnalysisResult { val container = createContainer( project, files, trace, configuration, packagePartProvider, declarationProviderFactory, CompilerEnvironment, - sourceModuleSearchScope + sourceModuleSearchScope, klibList ) val module = container.get() @@ -124,7 +133,8 @@ object TopDownAnalyzerFacadeForJVM { packagePartProvider: (GlobalSearchScope) -> PackagePartProvider, declarationProviderFactory: (StorageManager, Collection) -> DeclarationProviderFactory, targetEnvironment: TargetEnvironment = CompilerEnvironment, - sourceModuleSearchScope: GlobalSearchScope = newModuleSearchScope(project, files) + sourceModuleSearchScope: GlobalSearchScope = newModuleSearchScope(project, files), + klibList: List = emptyList() ): ComponentProvider { val jvmTarget = configuration.get(JVMConfigurationKeys.JVM_TARGET, JvmTarget.DEFAULT) val languageVersionSettings = configuration.languageVersionSettings @@ -233,9 +243,11 @@ object TopDownAnalyzerFacadeForJVM { extension.getPackageFragmentProvider(project, module, storageManager, trace, null, lookupTracker) } + val klibModules = getKlibModules(klibList, dependencyModule) + // TODO: remove dependencyModule from friends module.setDependencies( - listOfNotNull(module, dependencyModule, fallbackBuiltIns), + listOfNotNull(module, dependencyModule, fallbackBuiltIns) + klibModules, if (dependencyModule != null) setOf(dependencyModule) else emptySet() ) module.initialize( @@ -287,3 +299,41 @@ object TopDownAnalyzerFacadeForJVM { } } } + +// From serialization.js....klib.kt + +val jvmFactories = KlibMetadataFactories( + { storageManager -> JvmBuiltIns(storageManager, JvmBuiltIns.Kind.FROM_DEPENDENCIES) }, + NullFlexibleTypeDeserializer +) + +private fun getKlibModules(klibList: List, dependencyModule: ModuleDescriptorImpl?): List { + val descriptorMap = mutableMapOf() + return klibList.map { library -> + descriptorMap.getOrPut(library.libraryName) { getModuleDescriptorByLibrary(library, descriptorMap, dependencyModule) } + } +} + +private fun getModuleDescriptorByLibrary( + current: KotlinLibrary, mapping: + Map, + dependencyModule: ModuleDescriptorImpl? +): ModuleDescriptorImpl { + val module = jvmFactories.DefaultDeserializedDescriptorFactory.createDescriptorOptionalBuiltIns( + current, + LanguageVersionSettingsImpl.DEFAULT, + LockBasedStorageManager.NO_LOCKS, + null, + packageAccessHandler = null // TODO: This is a speed optimization used by Native. Don't bother for now. + ) + + val dependencies = current.manifestProperties.propertyList(KLIB_PROPERTY_DEPENDS, escapeInQuotes = true).mapNotNull { + mapping[it] ?: run { + assert(it in jvmLibrariesProvidedByDefault) { "Unknown library $it" } + null + } + } + + module.setDependencies(listOf(module) + dependencies + listOfNotNull(dependencyModule)) + return module +} diff --git a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmBackendFacade.kt b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmBackendFacade.kt index 8aaa2530e5c..b5903a57dee 100644 --- a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmBackendFacade.kt +++ b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/JvmBackendFacade.kt @@ -12,6 +12,9 @@ import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.jvm.codegen.ClassCodegen import org.jetbrains.kotlin.backend.jvm.lower.MultifileFacadeFileEntry import org.jetbrains.kotlin.codegen.state.GenerationState +import org.jetbrains.kotlin.descriptors.konan.KlibModuleOrigin +import org.jetbrains.kotlin.ir.backend.jvm.serialization.EmptyLoggingContext +import org.jetbrains.kotlin.ir.backend.jvm.serialization.JvmIrLinker import org.jetbrains.kotlin.ir.backend.jvm.serialization.JvmMangler import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.ir.declarations.IrModuleFragment @@ -43,11 +46,26 @@ object JvmBackendFacade { } } - val irProviders = generateTypicalIrProviderList( - psi2irContext.moduleDescriptor, psi2irContext.irBuiltIns, psi2irContext.symbolTable, - extensions = extensions + val stubGenerator = DeclarationStubGenerator( + psi2irContext.moduleDescriptor, psi2irContext.symbolTable, psi2irContext.irBuiltIns.languageVersionSettings, extensions ) - val irModuleFragment = psi2ir.generateModuleFragment(psi2irContext, files, irProviders = irProviders, expectDescriptorToSymbol = null) + val deserializer = JvmIrLinker( + psi2irContext.moduleDescriptor, EmptyLoggingContext, psi2irContext.irBuiltIns, psi2irContext.symbolTable, stubGenerator + ) + psi2irContext.moduleDescriptor.allDependencyModules.filter { it.getCapability(KlibModuleOrigin.CAPABILITY) != null }.forEach { + deserializer.deserializeIrModuleHeader(it) + } + val irProviders = listOf(deserializer, stubGenerator) + stubGenerator.setIrProviders(irProviders) + + val irModuleFragment = psi2ir.generateModuleFragment( + psi2irContext, files, + irProviders = irProviders, + expectDescriptorToSymbol = null + ) + // We need to compile all files we reference in Klibs + irModuleFragment.files.addAll(deserializer.getAllIrFiles()) + doGenerateFilesInternal( state, irModuleFragment, psi2irContext.symbolTable, psi2irContext.sourceManager, phaseConfig, irProviders, extensions ) diff --git a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/ClassCodegen.kt b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/ClassCodegen.kt index b4c14c89d0e..6ed474f283c 100644 --- a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/ClassCodegen.kt +++ b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/codegen/ClassCodegen.kt @@ -282,7 +282,7 @@ open class ClassCodegen protected constructor( if (entry is MultifileFacadeFileEntry) { return entry.partFiles.flatMap { it.loadSourceFilesInfo() } } - return listOf(File(context.psiSourceManager.getFileEntry(this)!!.name)) + return listOfNotNull(context.psiSourceManager.getFileEntry(this)?.let { File(it.name) }) } companion object { diff --git a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/FileClassLowering.kt b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/FileClassLowering.kt index 5654a32a28c..24b2d4f1816 100644 --- a/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/FileClassLowering.kt +++ b/compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/FileClassLowering.kt @@ -21,18 +21,26 @@ import org.jetbrains.kotlin.backend.common.ir.createImplicitParameterDeclaration import org.jetbrains.kotlin.backend.common.phaser.makeIrModulePhase import org.jetbrains.kotlin.backend.jvm.JvmBackendContext import org.jetbrains.kotlin.codegen.AsmUtil +import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.ClassKind import org.jetbrains.kotlin.descriptors.Modality import org.jetbrains.kotlin.descriptors.Visibilities +import org.jetbrains.kotlin.fileClasses.JvmFileClassInfo import org.jetbrains.kotlin.fileClasses.JvmFileClassUtil +import org.jetbrains.kotlin.fileClasses.JvmSimpleFileClassInfo import org.jetbrains.kotlin.ir.declarations.* import org.jetbrains.kotlin.ir.declarations.impl.IrClassImpl import org.jetbrains.kotlin.ir.descriptors.WrappedClassDescriptor import org.jetbrains.kotlin.ir.symbols.impl.IrClassSymbolImpl +import org.jetbrains.kotlin.ir.util.NaiveSourceBasedFileEntryImpl import org.jetbrains.kotlin.ir.util.fqNameWhenAvailable +import org.jetbrains.kotlin.load.kotlin.PackagePartClassUtils +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.psi2ir.PsiSourceManager import org.jetbrains.kotlin.resolve.jvm.JvmClassName import org.jetbrains.kotlin.resolve.source.KotlinSourceElement +import java.io.File import java.util.* internal val fileClassPhase = makeIrModulePhase( @@ -71,10 +79,21 @@ private class FileClassLowering(val context: JvmBackendContext) : FileLoweringPa private fun createFileClass(irFile: IrFile, fileClassMembers: List): IrClass { val fileEntry = irFile.fileEntry - val ktFile = context.psiSourceManager.getKtFile(fileEntry as PsiSourceManager.PsiFileEntry) - ?: throw AssertionError("Unexpected file entry: $fileEntry") - val fileClassInfo = JvmFileClassUtil.getFileClassInfoNoResolve(ktFile) - val descriptor = WrappedClassDescriptor(sourceElement = KotlinSourceElement(ktFile)) + val fileClassInfo: JvmFileClassInfo? + val descriptor: ClassDescriptor + when (fileEntry) { + is PsiSourceManager.PsiFileEntry -> { + val ktFile = context.psiSourceManager.getKtFile(fileEntry as PsiSourceManager.PsiFileEntry) + ?: throw AssertionError("Unexpected file entry: $fileEntry") + fileClassInfo = JvmFileClassUtil.getFileClassInfoNoResolve(ktFile) + descriptor = WrappedClassDescriptor(sourceElement = KotlinSourceElement(ktFile)) + } + is NaiveSourceBasedFileEntryImpl -> { + fileClassInfo = JvmSimpleFileClassInfo(PackagePartClassUtils.getPackagePartFqName(irFile.fqName, fileEntry.name), false) + descriptor = WrappedClassDescriptor() + } + else -> error("unknown kind of file entry: $fileEntry") + } return IrClassImpl( 0, fileEntry.maxOffset, IrDeclarationOrigin.FILE_CLASS, diff --git a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/IrUtils.kt b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/IrUtils.kt index 90c502dee77..0dd71c216ca 100644 --- a/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/IrUtils.kt +++ b/compiler/ir/ir.tree/src/org/jetbrains/kotlin/ir/util/IrUtils.kt @@ -6,6 +6,7 @@ package org.jetbrains.kotlin.ir.util import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.descriptors.impl.DeclarationDescriptorVisitorEmptyBodies import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.IrStatement import org.jetbrains.kotlin.ir.declarations.* @@ -21,6 +22,7 @@ import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.SpecialNames import org.jetbrains.kotlin.psi.psiUtil.endOffset import org.jetbrains.kotlin.psi.psiUtil.startOffset +import org.jetbrains.kotlin.resolve.DescriptorUtils import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe import org.jetbrains.kotlin.resolve.source.PsiSourceElement import org.jetbrains.kotlin.types.KotlinType @@ -632,3 +634,37 @@ val IrFunctionReference.typeSubstitutionMap: Map val IrFunctionAccessExpression.typeSubstitutionMap: Map get() = getTypeSubstitutionMap(symbol.owner) + +fun SymbolTable.referenceMember(descriptor: DeclarationDescriptor): IrSymbol = + descriptor.accept( + object : DeclarationDescriptorVisitorEmptyBodies() { + override fun visitClassDescriptor(descriptor: ClassDescriptor, data: Unit?) = + if (DescriptorUtils.isEnumEntry(descriptor)) + referenceEnumEntry(descriptor) + else + referenceClass(descriptor) + + override fun visitConstructorDescriptor(constructorDescriptor: ConstructorDescriptor, data: Unit?) = + referenceConstructor(descriptor as ClassConstructorDescriptor) + + override fun visitFunctionDescriptor(descriptor: FunctionDescriptor, data: Unit?) = referenceSimpleFunction(descriptor) + + override fun visitPropertyDescriptor(descriptor: PropertyDescriptor, data: Unit?) = referenceProperty(descriptor) + + override fun visitTypeParameterDescriptor(descriptor: TypeParameterDescriptor, data: Unit?) = referenceTypeParameter(descriptor) + + override fun visitTypeAliasDescriptor(descriptor: TypeAliasDescriptor, data: Unit?) = referenceTypeAlias(descriptor) + + override fun visitDeclarationDescriptor(descriptor: DeclarationDescriptor?, data: Unit?): IrSymbol { + throw AssertionError("Unexpected member descriptor: $descriptor") + } + }, + null + ) + +fun SymbolTable.findOrDeclareExternalPackageFragment(descriptor: PackageFragmentDescriptor) = + referenceExternalPackageFragment(descriptor).also { + if (!it.isBound) { + declareExternalPackageFragment(descriptor) + } + }.owner diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt index 3b8e92ac0cc..36aece2a114 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/DeserializeDescriptorReference.kt @@ -27,11 +27,11 @@ import org.jetbrains.kotlin.serialization.deserialization.descriptors.Deserializ abstract class DescriptorReferenceDeserializer( val currentModule: ModuleDescriptor, val mangler: KotlinMangler, - val builtIns: IrBuiltIns, + val builtIns: IrBuiltIns?, val resolvedForwardDeclarations: MutableMap ) : DescriptorUniqIdAware { - protected open fun resolveSpecialDescriptor(fqn: FqName) = builtIns.builtIns.getBuiltInClassByFqName(fqn) + protected open fun resolveSpecialDescriptor(fqn: FqName) = builtIns!!.builtIns.getBuiltInClassByFqName(fqn) open fun checkIfSpecialDescriptorId(id: Long) = with(mangler) { id.isSpecial } protected open fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor) = diff --git a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt index 802ef2c8f9a..01777e8e2d4 100644 --- a/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt +++ b/compiler/ir/serialization.common/src/org/jetbrains/kotlin/backend/common/serialization/KotlinIrLinker.kt @@ -46,7 +46,8 @@ abstract class KotlinIrLinker( val symbolTable: SymbolTable, private val exportedDependencies: List, private val forwardModuleDescriptor: ModuleDescriptor?, - mangler: KotlinMangler + mangler: KotlinMangler, + private val tolerateNonKlibDescriptors: Boolean = false ) : DescriptorUniqIdAware, IrDeserializer { private val expectUniqIdToActualUniqId = mutableMapOf() @@ -133,7 +134,9 @@ abstract class KotlinIrLinker( private val moduleDeserializationState = DeserializationState.ModuleDeserializationState(this) val moduleReversedFileIndex = mutableMapOf() private val moduleDependencies by lazy { - moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.map { deserializersForModules[it]!! } + moduleDescriptor.allDependencyModules.filter { it != moduleDescriptor }.mapNotNull { + deserializersForModules[it] ?: if (tolerateNonKlibDescriptors) null else error("Module without deserializer $it") + } } // This is a heavy initializer @@ -562,12 +565,21 @@ abstract class KotlinIrLinker( } val descriptorUniqId = topLevelDescriptor.getUniqId() - ?: error("Could not get descriptor uniq id for $topLevelDescriptor") + ?: if (tolerateNonKlibDescriptors) { + return null + } else { + error("Could not get descriptor uniq id for $topLevelDescriptor") + } val topLevelKey = UniqId(descriptorUniqId) val moduleOfOrigin = topLevelDescriptor.module - val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: error("No module deserializer found for $moduleOfOrigin") + val moduleDeserializer = deserializersForModules[moduleOfOrigin] ?: + if (tolerateNonKlibDescriptors) { + return null + } else { + error("No module deserializer found for $moduleOfOrigin") + } moduleDeserializer.addModuleReachableTopLevel(topLevelKey) @@ -697,6 +709,10 @@ abstract class KotlinIrLinker( fun deserializeOnlyHeaderModule(moduleDescriptor: ModuleDescriptor): IrModuleFragment = deserializeIrModuleHeader(moduleDescriptor, DeserializationStrategy.ONLY_DECLARATION_HEADERS) + + fun getAllIrFiles(): List { + return deserializersForModules.values.flatMap { it.module.files } + } } enum class DeserializationStrategy(val needBodies: Boolean, val explicitlyExported: Boolean, val theWholeWorld: Boolean) { diff --git a/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/JvmLibraryResolver.kt b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/JvmLibraryResolver.kt new file mode 100644 index 00000000000..a147887502d --- /dev/null +++ b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/JvmLibraryResolver.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.jvm + +import org.jetbrains.kotlin.konan.CompilerVersion +import org.jetbrains.kotlin.konan.file.File +import org.jetbrains.kotlin.library.KotlinAbiVersion +import org.jetbrains.kotlin.library.KotlinLibrary +import org.jetbrains.kotlin.library.KotlinLibraryProperResolverWithAttributes +import org.jetbrains.kotlin.library.UnresolvedLibrary +import org.jetbrains.kotlin.library.impl.createKotlinLibraryComponents +import org.jetbrains.kotlin.library.resolver.KotlinLibraryResolveResult +import org.jetbrains.kotlin.library.resolver.impl.libraryResolver +import org.jetbrains.kotlin.util.Logger + +val jvmLibrariesProvidedByDefault = setOf("stdlib", "kotlin") + +class JvmLibraryResolver( + repositories: List, + directLibs: List, + knownAbiVersions: List?, + knownCompilerVersions: List?, + distributionKlib: String?, + localKotlinDir: String?, + skipCurrentDir: Boolean, + logger: Logger +) : KotlinLibraryProperResolverWithAttributes( + repositories, + directLibs, + knownAbiVersions, + knownCompilerVersions, + distributionKlib, + localKotlinDir, + skipCurrentDir, + logger, + emptyList() +) { + // Stick with the default KotlinLibrary for now. + override fun libraryComponentBuilder(file: File, isDefault: Boolean) = createKotlinLibraryComponents(file, isDefault) + + // We do not need stdlib in klib form. + override fun isProvidedByDefault(unresolved: UnresolvedLibrary): Boolean = + unresolved.path in jvmLibrariesProvidedByDefault +} + +// TODO: This is a temporary set of library resolver policies for jvm compiler. +fun jvmResolveLibraries(libraries: List, logger: Logger): KotlinLibraryResolveResult { + val unresolvedLibraries = libraries.map { UnresolvedLibrary(it, null) } + val libraryAbsolutePaths = libraries.map { File(it).absolutePath } + // Configure the resolver to only work with absolute paths for now. + val libraryResolver = JvmLibraryResolver( + repositories = emptyList(), + directLibs = libraryAbsolutePaths, + knownAbiVersions = listOf(KotlinAbiVersion.CURRENT), + knownCompilerVersions = emptyList(), + distributionKlib = null, + localKotlinDir = null, + skipCurrentDir = false, + logger = logger + ).libraryResolver() + val resolvedLibraries = + libraryResolver.resolveWithDependencies( + unresolvedLibraries = unresolvedLibraries, + noStdLib = true, + noDefaultLibs = true, + noEndorsedLibs = true + ) + return resolvedLibraries +} \ No newline at end of file diff --git a/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorReferenceDeserializer.kt b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorReferenceDeserializer.kt new file mode 100644 index 00000000000..ac8bfcbc176 --- /dev/null +++ b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorReferenceDeserializer.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.jvm.serialization + +import org.jetbrains.kotlin.backend.common.serialization.DescriptorReferenceDeserializer +import org.jetbrains.kotlin.backend.common.serialization.DescriptorUniqIdAware +import org.jetbrains.kotlin.descriptors.ClassDescriptor +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.name.FqName + +class JvmDescriptorReferenceDeserializer(currentModule: ModuleDescriptor, private val uniqIdAware: DescriptorUniqIdAware) : + DescriptorReferenceDeserializer(currentModule, JvmMangler, builtIns = null, resolvedForwardDeclarations = mutableMapOf()), + DescriptorUniqIdAware by uniqIdAware { + + override fun resolveSpecialDescriptor(fqn: FqName): ClassDescriptor { + error("Should never be called") + } + + override fun checkIfSpecialDescriptorId(id: Long) = false + + override fun getDescriptorIdOrNull(descriptor: DeclarationDescriptor): Long? = null +} \ No newline at end of file diff --git a/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorUniqIdAware.kt b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorUniqIdAware.kt new file mode 100644 index 00000000000..a5efa529e68 --- /dev/null +++ b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmDescriptorUniqIdAware.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.jvm.serialization + +import org.jetbrains.kotlin.backend.common.serialization.DescriptorUniqIdAware +import org.jetbrains.kotlin.backend.common.serialization.tryGetExtension +import org.jetbrains.kotlin.descriptors.* +import org.jetbrains.kotlin.ir.declarations.IrDeclaration +import org.jetbrains.kotlin.ir.declarations.IrDeclarationParent +import org.jetbrains.kotlin.ir.util.* +import org.jetbrains.kotlin.library.metadata.KlibMetadataProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.descriptors.* + +class JvmDescriptorUniqIdAware(val symbolTable: SymbolTable, val stubGenerator: DeclarationStubGenerator) : DescriptorUniqIdAware { + override fun DeclarationDescriptor.getUniqId(): Long? = + when (this) { + is DeserializedClassDescriptor -> this.classProto.tryGetExtension(KlibMetadataProtoBuf.classUniqId)?.index + ?: referenceAndHash(this) + is DeserializedSimpleFunctionDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.functionUniqId)?.index + ?: referenceAndHash(this) + is DeserializedPropertyDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.propertyUniqId)?.index + ?: referenceAndHash(this) + is DeserializedClassConstructorDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.constructorUniqId)?.index + ?: referenceAndHash(this) + is DeserializedTypeParameterDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.typeParamUniqId)?.index + ?: referenceAndHash(this) + is DeserializedTypeAliasDescriptor -> this.proto.tryGetExtension(KlibMetadataProtoBuf.typeAliasUniqId)?.index + ?: referenceAndHash(this) + else -> referenceAndHash(this) + } + + private fun referenceAndHash(descriptor: DeclarationDescriptor): Long? = + if (descriptor is CallableMemberDescriptor && descriptor.kind === CallableMemberDescriptor.Kind.FAKE_OVERRIDE) + null + else with(JvmMangler) { + referenceWithParents(descriptor).hashedMangle + } + + + private fun referenceWithParents(descriptor: DeclarationDescriptor): IrDeclaration { + val original = descriptor.original + val result = referenceOrDeclare(original) + var currentDescriptor = original + var current = result + // If current is a lazy declaration, the parent may already be set. + while (current.parent == null) { + val nextDescriptor = when { + currentDescriptor is TypeParameterDescriptor && currentDescriptor.containingDeclaration is PropertyDescriptor -> { + val property = currentDescriptor.containingDeclaration as PropertyDescriptor + // No way to choose between getter and setter by descriptor alone :( + property.getter ?: property.setter!! + } + else -> + currentDescriptor.containingDeclaration!! + } + if (nextDescriptor is PackageFragmentDescriptor) { + current.parent = symbolTable.findOrDeclareExternalPackageFragment(nextDescriptor) + break + } else { + val next = referenceOrDeclare(nextDescriptor) + current.parent = next as IrDeclarationParent + currentDescriptor = nextDescriptor + current = next + } + } + return result + } + + private fun referenceOrDeclare(descriptor: DeclarationDescriptor): IrDeclaration = + symbolTable.referenceMember(descriptor).also { + if (!it.isBound) { + stubGenerator.getDeclaration(it) + } + }.owner as IrDeclaration +} + +// May be needed in the future +// +//fun DeclarationDescriptor.willBeEliminatedInLowerings(): Boolean = +// isAnnotationConstructor() || +// (this is PropertyAccessorDescriptor && +// correspondingProperty.hasJvmFieldAnnotation()) diff --git a/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmIrLinker.kt b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmIrLinker.kt new file mode 100644 index 00000000000..cece89a684c --- /dev/null +++ b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/JvmIrLinker.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.jvm.serialization + +import org.jetbrains.kotlin.backend.common.LoggingContext +import org.jetbrains.kotlin.backend.common.serialization.* +import org.jetbrains.kotlin.descriptors.DeclarationDescriptor +import org.jetbrains.kotlin.descriptors.DeclarationDescriptorWithVisibility +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.Visibilities +import org.jetbrains.kotlin.descriptors.konan.kotlinLibrary +import org.jetbrains.kotlin.ir.descriptors.IrBuiltIns +import org.jetbrains.kotlin.ir.util.DeclarationStubGenerator +import org.jetbrains.kotlin.ir.util.SymbolTable +import org.jetbrains.kotlin.ir.util.UniqId +import org.jetbrains.kotlin.load.java.JavaVisibilities +import org.jetbrains.kotlin.load.java.descriptors.JavaClassDescriptor +import org.jetbrains.kotlin.resolve.descriptorUtil.isPublishedApi + +class JvmIrLinker( + currentModule: ModuleDescriptor, + logger: LoggingContext, + builtIns: IrBuiltIns, + symbolTable: SymbolTable, + stubGenerator: DeclarationStubGenerator +) : KotlinIrLinker(logger, builtIns, symbolTable, emptyList(), null, JvmMangler, + // TODO: Use protected shouldBeDeserialized() ? + tolerateNonKlibDescriptors = true), + DescriptorUniqIdAware { + + override val descriptorReferenceDeserializer = + JvmDescriptorReferenceDeserializer(currentModule, JvmDescriptorUniqIdAware(symbolTable, stubGenerator)) + override fun DeclarationDescriptor.getUniqId() = with(descriptorReferenceDeserializer) { + getUniqId() + } + + override fun handleNoModuleDeserializerFound(key: UniqId): DeserializationState<*> { + return globalDeserializationState // !!!!!! Wrong, as external references will all have UniqId.NONE + } + + override fun reader(moduleDescriptor: ModuleDescriptor, fileIndex: Int, uniqId: UniqId) = + moduleDescriptor.kotlinLibrary.irDeclaration(uniqId.index, fileIndex) + + override fun readSymbol(moduleDescriptor: ModuleDescriptor, fileIndex: Int, symbolIndex: Int) = + moduleDescriptor.kotlinLibrary.symbol(symbolIndex, fileIndex) + + override fun readType(moduleDescriptor: ModuleDescriptor, fileIndex: Int, typeIndex: Int) = + moduleDescriptor.kotlinLibrary.type(typeIndex, fileIndex) + + override fun readString(moduleDescriptor: ModuleDescriptor, fileIndex: Int, stringIndex: Int) = + moduleDescriptor.kotlinLibrary.string(stringIndex, fileIndex) + + override fun readBody(moduleDescriptor: ModuleDescriptor, fileIndex: Int, bodyIndex: Int) = + moduleDescriptor.kotlinLibrary.body(bodyIndex, fileIndex) + + override fun readFile(moduleDescriptor: ModuleDescriptor, fileIndex: Int) = + moduleDescriptor.kotlinLibrary.file(fileIndex) + + override fun readFileCount(moduleDescriptor: ModuleDescriptor) = + moduleDescriptor.kotlinLibrary.fileCount() + + override fun checkAccessibility(declarationDescriptor: DeclarationDescriptor): Boolean { + require(declarationDescriptor is DeclarationDescriptorWithVisibility) + return declarationDescriptor.isPublishedApi() || + declarationDescriptor.visibility.let { + it.isPublicAPI || it == Visibilities.INTERNAL || + it == JavaVisibilities.PACKAGE_VISIBILITY || it == JavaVisibilities.PROTECTED_AND_PACKAGE + } + } + + private val ModuleDescriptor.userName get() = kotlinLibrary.libraryFile.absolutePath +} \ No newline at end of file diff --git a/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/Utils.kt b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/Utils.kt new file mode 100644 index 00000000000..ea6ff1eddbd --- /dev/null +++ b/compiler/ir/serialization.jvm/src/org/jetbrains/kotlin/ir/backend/jvm/serialization/Utils.kt @@ -0,0 +1,14 @@ +/* + * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors. + * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file. + */ + +package org.jetbrains.kotlin.ir.backend.jvm.serialization + +import org.jetbrains.kotlin.backend.common.LoggingContext + +object EmptyLoggingContext: LoggingContext { + override var inVerbosePhase = false + + override fun log(message: () -> String) {} +} \ No newline at end of file diff --git a/compiler/testData/cli/jvm/extraHelp.out b/compiler/testData/cli/jvm/extraHelp.out index 06ddf0df617..511c5075161 100644 --- a/compiler/testData/cli/jvm/extraHelp.out +++ b/compiler/testData/cli/jvm/extraHelp.out @@ -42,6 +42,7 @@ where advanced options include: in the interface (annotating an existing method can break binary compatibility) -Xjvm-default=compatibility Allow usages of @JvmDefault; generate a compatibility accessor in the 'DefaultImpls' class in addition to the interface method + -Xklib= Paths to cross-platform libraries in .klib format -Xno-call-assertions Don't generate not-null assertions for arguments of platform types -Xno-exception-on-explicit-equals-for-boxed-null Do not throw NPE on explicit 'equals' call for null receiver of platform boxed primitive type diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt index 573ea095bd6..d204f852e4d 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/codegen/GenerationUtils.kt @@ -26,27 +26,36 @@ import org.jetbrains.kotlin.asJava.finder.JavaElementFinder import org.jetbrains.kotlin.backend.common.phaser.PhaseConfig import org.jetbrains.kotlin.backend.jvm.JvmIrCodegenFactory import org.jetbrains.kotlin.backend.jvm.jvmPhases +import org.jetbrains.kotlin.builtins.jvm.JvmBuiltIns import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.output.writeAllTo +import org.jetbrains.kotlin.cli.js.messageCollectorLogger import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM import org.jetbrains.kotlin.codegen.state.GenerationState -import org.jetbrains.kotlin.config.CommonConfigurationKeys -import org.jetbrains.kotlin.config.CompilerConfiguration -import org.jetbrains.kotlin.config.JVMConfigurationKeys -import org.jetbrains.kotlin.config.languageVersionSettings +import org.jetbrains.kotlin.config.* +import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.fir.backend.Fir2IrConverter import org.jetbrains.kotlin.fir.builder.RawFirBuilder import org.jetbrains.kotlin.fir.createSession import org.jetbrains.kotlin.fir.resolve.firProvider import org.jetbrains.kotlin.fir.resolve.impl.FirProviderImpl import org.jetbrains.kotlin.fir.resolve.transformers.FirTotalResolveTransformer +import org.jetbrains.kotlin.ir.backend.jvm.jvmLibrariesProvidedByDefault +import org.jetbrains.kotlin.ir.backend.jvm.jvmResolveLibraries +import org.jetbrains.kotlin.konan.properties.propertyList +import org.jetbrains.kotlin.konan.util.KlibMetadataFactories +import org.jetbrains.kotlin.library.KLIB_PROPERTY_DEPENDS +import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.load.kotlin.PackagePartProvider import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.AnalyzingUtils import org.jetbrains.kotlin.resolve.BindingTrace import org.jetbrains.kotlin.resolve.lazy.JvmResolveUtil +import org.jetbrains.kotlin.serialization.konan.NullFlexibleTypeDeserializer +import org.jetbrains.kotlin.storage.LockBasedStorageManager +import org.jetbrains.kotlin.util.DummyLogger import java.io.File object GenerationUtils { @@ -153,8 +162,17 @@ object GenerationUtils { packagePartProvider: (GlobalSearchScope) -> PackagePartProvider, trace: BindingTrace ): GenerationState { + val logger = configuration.get(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY)?.let { messageCollectorLogger(it) } + ?: DummyLogger + val resolvedKlibs = configuration.get(JVMConfigurationKeys.KLIB_PATHS)?.let { klibPaths -> + jvmResolveLibraries(klibPaths, logger) + } + val analysisResult = - JvmResolveUtil.analyzeAndCheckForErrors(project, files, configuration, packagePartProvider, trace) + JvmResolveUtil.analyzeAndCheckForErrors( + project, files, configuration, packagePartProvider, trace, + klibList = resolvedKlibs?.getFullList() ?: emptyList() + ) analysisResult.throwIfError() /* Currently Kapt3 only works with the old JVM backend, so disable IR for everything except actual bytecode generation. */ diff --git a/compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt b/compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt index a5e07f0c97e..204c8451c9a 100644 --- a/compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt +++ b/compiler/tests-common/tests/org/jetbrains/kotlin/resolve/lazy/JvmResolveUtil.kt @@ -26,6 +26,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.NoScopeRecordCliBindingTrace import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.container.ComponentProvider +import org.jetbrains.kotlin.library.KotlinLibrary import org.jetbrains.kotlin.load.kotlin.PackagePartProvider import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.AnalyzingUtils @@ -62,7 +63,8 @@ object JvmResolveUtil { files: Collection, configuration: CompilerConfiguration, packagePartProvider: (GlobalSearchScope) -> PackagePartProvider, - trace: BindingTrace = CliBindingTrace() + trace: BindingTrace = CliBindingTrace(), + klibList: List = emptyList() ): AnalysisResult { for (file in files) { try { @@ -72,7 +74,7 @@ object JvmResolveUtil { } } - return analyze(project, files, configuration, packagePartProvider, trace).apply { + return analyze(project, files, configuration, packagePartProvider, trace, klibList).apply { try { AnalyzingUtils.throwExceptionOnErrors(bindingContext) } catch (e: Exception) { @@ -102,10 +104,12 @@ object JvmResolveUtil { files: Collection, configuration: CompilerConfiguration, packagePartProviderFactory: (GlobalSearchScope) -> PackagePartProvider, - trace: BindingTrace = CliBindingTrace() + trace: BindingTrace = CliBindingTrace(), + klibList: List = emptyList() ): AnalysisResult { return TopDownAnalyzerFacadeForJVM.analyzeFilesWithJavaIntegration( - project, files, trace, configuration, packagePartProviderFactory + project, files, trace, configuration, packagePartProviderFactory, + klibList = klibList ) } }