From bfb7b21472042884e75c7c6a5956dedb1618ff91 Mon Sep 17 00:00:00 2001 From: Alexander Udalov Date: Wed, 16 Nov 2016 11:47:05 +0300 Subject: [PATCH] Load definitions of symbols from .kotlin_metadata files Extract AbstractDeserializedPackageFragmentProvider out of JvmBuiltInsPackageFragmentProvider and implement it a little bit differently in MetadataPackageFragmentProvider. The main difference is in how the package fragment scope is constructed: for built-ins, it's just a single scope that loads everything from one protobuf message. For metadata, package scope can consist of many files, some of which store information about classes and others are similar to package parts on JVM, so a ChainedMemberScope instance is created. Introduce a bunch of interfaces/methods to deliver the needed behavior to the 'deserialization' module which is not JVM-specific and does not depend on the compiler code: MetadataFinderFactory, PackagePartProvider#findMetadataPackageParts, KotlinMetadataFinder#findMetadata. Note that these declarations are currently only implemented in the compiler; no metadata package parts/fragments will be found in IDE or reflection --- .../serialization/MetadataSerializer.kt | 10 +- .../jvm/compiler/JvmCliVirtualFileFinder.kt | 7 + .../jvm/compiler/JvmPackagePartProvider.kt | 14 +- .../cli/jvm/compiler/KotlinCoreEnvironment.kt | 5 +- .../kotlin/JvmVirtualFileFinderFactory.kt | 2 +- .../IncrementalPackagePartProvider.kt | 3 + .../analyzer/common/DefaultAnalyzerFacade.kt | 16 ++- .../load/kotlin/MetadataFinderFactory.kt | 24 ++++ .../JvmBuiltInsPackageFragmentProvider.kt | 26 +--- .../kotlin/load/kotlin/ModuleMapping.kt | 29 ++-- .../reflect/ReflectKotlinClassFinder.kt | 3 + .../reflect/RuntimePackagePartProvider.kt | 3 + .../kotlin/descriptors/PackagePartProvider.kt | 7 + ...ractDeserializedPackageFragmentProvider.kt | 45 ++++++ .../DeserializedPackageFragment.kt | 17 ++- .../deserialization/KotlinMetadataFinder.kt | 7 + .../MetadataPackageFragmentProvider.kt | 135 ++++++++++++++++++ .../caches/resolve/IDEPackagePartProvider.kt | 15 +- .../DeserializerForClassfileDecompiler.kt | 3 + .../kotlin/idea/vfilefinder/fileFinders.kt | 3 + idea/src/META-INF/plugin.xml | 3 + 21 files changed, 320 insertions(+), 57 deletions(-) create mode 100644 compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt create mode 100644 core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/AbstractDeserializedPackageFragmentProvider.kt create mode 100644 core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt diff --git a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt index d19f4061976..7dca39927d5 100644 --- a/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt +++ b/compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt @@ -23,6 +23,7 @@ import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity +import org.jetbrains.kotlin.cli.jvm.compiler.JvmPackagePartProvider import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.codegen.JvmCodegenUtil import org.jetbrains.kotlin.codegen.serializeToByteArray @@ -41,6 +42,7 @@ import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension +import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment.Companion.METADATA_FILE_EXTENSION import org.jetbrains.kotlin.serialization.jvm.JvmPackageTable import java.io.ByteArrayOutputStream import java.io.DataOutputStream @@ -63,7 +65,9 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { val analyzer = AnalyzerWithCompilerReport(messageCollector) analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer { - override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) + override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) { + _, content -> JvmPackagePartProvider(environment, content.moduleContentScope) + } }) if (analyzer.hasErrors()) return @@ -193,8 +197,4 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) { destFile.writeBytes(stream.toByteArray()) } } - - companion object { - private val METADATA_FILE_EXTENSION = ".kotlin_metadata" - } } diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt index 115cfe33a37..7b56d301553 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt @@ -25,6 +25,7 @@ import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment import org.jetbrains.kotlin.utils.addToStdlib.check import java.io.InputStream @@ -35,6 +36,12 @@ class JvmCliVirtualFileFinder( override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? = findBinaryClass(classId, classId.relativeClassName.asString().replace('.', '$') + ".class") + override fun findMetadata(classId: ClassId): InputStream? { + assert(!classId.isNestedClass) { "Nested classes are not supported here: $classId" } + + return findBinaryClass(classId, classId.shortClassName.asString() + MetadataPackageFragment.METADATA_FILE_EXTENSION)?.inputStream + } + override fun findBuiltInsData(packageFqName: FqName): InputStream? { // "" is just a made-up name // JvmDependenciesIndex requires the ClassId of the class which we're searching for, to cache the last request+result diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt index aa5b65d99f2..bc0f43207c6 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt @@ -22,6 +22,7 @@ import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot import org.jetbrains.kotlin.config.JVMConfigurationKeys import org.jetbrains.kotlin.descriptors.PackagePartProvider import org.jetbrains.kotlin.load.kotlin.ModuleMapping +import org.jetbrains.kotlin.load.kotlin.PackageParts import java.io.EOFException class JvmPackagePartProvider( @@ -38,11 +39,16 @@ class JvmPackagePartProvider( private val loadedModules: MutableList = SmartList() - @Synchronized - override fun findPackageParts(packageFqName: String): List { - processNotLoadedRelevantRoots(packageFqName) + override fun findPackageParts(packageFqName: String): List = + getPackageParts(packageFqName).flatMap(PackageParts::parts).distinct() - return loadedModules.flatMap { it.findPackageParts(packageFqName)?.parts ?: emptySet() }.distinct() + override fun findMetadataPackageParts(packageFqName: String): List = + getPackageParts(packageFqName).flatMap(PackageParts::metadataParts).distinct() + + @Synchronized + private fun getPackageParts(packageFqName: String): List { + processNotLoadedRelevantRoots(packageFqName) + return loadedModules.mapNotNull { it.findPackageParts(packageFqName) } } private fun processNotLoadedRelevantRoots(packageFqName: String) { diff --git a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt index 466f9695456..8fe8dc72609 100644 --- a/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt +++ b/compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt @@ -86,6 +86,7 @@ import org.jetbrains.kotlin.extensions.StorageComponentContainerContributor import org.jetbrains.kotlin.idea.KotlinFileType import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory import org.jetbrains.kotlin.load.kotlin.KotlinBinaryClassCache +import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.isValidJavaFqName @@ -160,7 +161,9 @@ class KotlinCoreEnvironment private constructor( (ServiceManager.getService(project, CoreJavaFileManager::class.java) as KotlinCliJavaFileManagerImpl).initIndex(rootsIndex) - project.registerService(JvmVirtualFileFinderFactory::class.java, JvmCliVirtualFileFinderFactory(rootsIndex)) + val finderFactory = JvmCliVirtualFileFinderFactory(rootsIndex) + project.registerService(MetadataFinderFactory::class.java, finderFactory) + project.registerService(JvmVirtualFileFinderFactory::class.java, finderFactory) ExpressionCodegenExtension.registerExtensionPoint(project) ClassBuilderInterceptorExtension.registerExtensionPoint(project) diff --git a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt index 27273c9e8a4..9cb76c6dd34 100644 --- a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt +++ b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt @@ -20,7 +20,7 @@ import com.intellij.openapi.components.ServiceManager import com.intellij.openapi.project.Project import com.intellij.psi.search.GlobalSearchScope -interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory { +interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory, MetadataFinderFactory { override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder object SERVICE { diff --git a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt index 8517b683f6c..411a87f13d8 100644 --- a/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt +++ b/compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt @@ -39,6 +39,9 @@ internal class IncrementalPackagePartProvider private constructor( parent.findPackageParts(packageFqName)).distinct() } + // TODO + override fun findMetadataPackageParts(packageFqName: String): List = TODO() + companion object { @JvmStatic fun create( diff --git a/compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt b/compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt index d4cd2e518e6..9beb9245fa5 100644 --- a/compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt +++ b/compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt @@ -16,6 +16,7 @@ package org.jetbrains.kotlin.analyzer.common +import com.intellij.openapi.components.ServiceManager import com.intellij.psi.search.GlobalSearchScope import org.jetbrains.kotlin.analyzer.* import org.jetbrains.kotlin.config.ApiVersion @@ -30,9 +31,11 @@ import org.jetbrains.kotlin.context.LazyResolveToken import org.jetbrains.kotlin.context.ModuleContext import org.jetbrains.kotlin.context.ProjectContext import org.jetbrains.kotlin.descriptors.PackagePartProvider +import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl import org.jetbrains.kotlin.frontend.di.configureModule import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.resolve.* @@ -40,6 +43,7 @@ import org.jetbrains.kotlin.resolve.lazy.FileScopeProviderImpl import org.jetbrains.kotlin.resolve.lazy.ResolveSession import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService +import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragmentProvider /** * A facade that is used to analyze platform independent modules in multi-platform projects. @@ -97,7 +101,12 @@ object DefaultAnalyzerFacade : AnalyzerFacade() { moduleContext, trace, declarationProviderFactory, moduleContentScope, targetEnvironment, packagePartProvider ) - return ResolverForModule(container.get().packageFragmentProvider, container) + val packageFragmentProviders = listOf( + container.get().packageFragmentProvider, + container.get() + ) + + return ResolverForModule(CompositePackageFragmentProvider(packageFragmentProviders), container) } private fun createContainerToResolveCommonCode( @@ -119,6 +128,11 @@ object DefaultAnalyzerFacade : AnalyzerFacade() { useImpl() useInstance(packagePartProvider) useInstance(declarationProviderFactory) + useImpl() + + val metadataFinderFactory = ServiceManager.getService(moduleContext.project, MetadataFinderFactory::class.java) + ?: error("No MetadataFinderFactory in project") + useInstance(metadataFinderFactory.create(moduleContentScope)) targetEnvironment.configure(this) useImpl() diff --git a/compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt b/compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt new file mode 100644 index 00000000000..f95410afa4b --- /dev/null +++ b/compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.load.kotlin + +import com.intellij.psi.search.GlobalSearchScope +import org.jetbrains.kotlin.serialization.deserialization.KotlinMetadataFinder + +interface MetadataFinderFactory { + fun create(scope: GlobalSearchScope): KotlinMetadataFinder +} diff --git a/core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt b/core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt index aad3cb1d8e3..aa3899382ab 100644 --- a/core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt +++ b/core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt @@ -18,15 +18,11 @@ package org.jetbrains.kotlin.builtins import org.jetbrains.kotlin.builtins.functions.BuiltInFictitiousFunctionClassFactory import org.jetbrains.kotlin.descriptors.ModuleDescriptor -import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor -import org.jetbrains.kotlin.descriptors.PackageFragmentProvider import org.jetbrains.kotlin.incremental.components.LookupTracker import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder import org.jetbrains.kotlin.name.FqName -import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.serialization.deserialization.* import org.jetbrains.kotlin.storage.StorageManager -import org.jetbrains.kotlin.utils.singletonOrEmptyList class JvmBuiltInsPackageFragmentProvider( storageManager: StorageManager, @@ -35,22 +31,12 @@ class JvmBuiltInsPackageFragmentProvider( notFoundClasses: NotFoundClasses, additionalClassPartsProvider: AdditionalClassPartsProvider, platformDependentDeclarationFilter: PlatformDependentDeclarationFilter -) : PackageFragmentProvider { - private lateinit var components: DeserializationComponents - - private val fragments = storageManager.createMemoizedFunctionWithNullableValues { fqName -> - finder.findBuiltInsData(fqName)?.let { inputStream -> - BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor, inputStream).apply { - components = this@JvmBuiltInsPackageFragmentProvider.components - } - } - } - +) : AbstractDeserializedPackageFragmentProvider(storageManager, finder, moduleDescriptor) { init { components = DeserializationComponents( storageManager, moduleDescriptor, - DeserializationConfiguration.Default, + DeserializationConfiguration.Default, // TODO DeserializedClassDataFinder(this), AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol), this, @@ -64,10 +50,10 @@ class JvmBuiltInsPackageFragmentProvider( ), notFoundClasses, additionalClassPartsProvider, platformDependentDeclarationFilter ) - } - override fun getPackageFragments(fqName: FqName): List = fragments(fqName).singletonOrEmptyList() - - override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection = emptySet() + override fun findPackage(fqName: FqName): DeserializedPackageFragment? = + finder.findBuiltInsData(fqName)?.let { inputStream -> + BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor, inputStream) + } } diff --git a/core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/ModuleMapping.kt b/core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/ModuleMapping.kt index e1cb2a4b907..f2ed5954aed 100644 --- a/core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/ModuleMapping.kt +++ b/core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/ModuleMapping.kt @@ -34,24 +34,20 @@ class ModuleMapping private constructor(val packageFqName2Parts: Map() - parseFrom.packagePartsList.forEach { - val packageParts = PackageParts(it.packageFqName) - packageFqNameParts.put(it.packageFqName, packageParts) - it.classNameList.forEach { - packageParts.parts.add(it) - } + val packageFqNameParts = hashMapOf().apply { + addParts(this, parseFrom.packagePartsList, PackageParts::parts) + addParts(this, parseFrom.metadataPartsList, PackageParts::metadataParts) } return ModuleMapping(packageFqNameParts, debugName ?: "") } @@ -62,6 +58,19 @@ class ModuleMapping private constructor(val packageFqName2Parts: Map, + partsList: List, + whichParts: (PackageParts) -> MutableSet + ) { + for (proto in partsList) { + PackageParts(proto.packageFqName).apply { + result.put(proto.packageFqName, this) + whichParts(this).addAll(proto.classNameList) + } + } + } } } diff --git a/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt b/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt index 0bccb489d7f..e0dd60aac27 100644 --- a/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt +++ b/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt @@ -36,6 +36,9 @@ class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinCla return findKotlinClass(javaClass.fqName?.asString() ?: return null) } + // TODO + override fun findMetadata(classId: ClassId): InputStream? = null + // TODO: load built-ins from classLoader override fun findBuiltInsData(packageFqName: FqName): InputStream? = null } diff --git a/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/RuntimePackagePartProvider.kt b/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/RuntimePackagePartProvider.kt index 8c9a66cf23f..8791fb68269 100644 --- a/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/RuntimePackagePartProvider.kt +++ b/core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/RuntimePackagePartProvider.kt @@ -40,4 +40,7 @@ class RuntimePackagePartProvider(private val classLoader: ClassLoader) : Package override fun findPackageParts(packageFqName: String): List { return module2Mapping.values.mapNotNull { it.findPackageParts(packageFqName) }.flatMap { it.parts }.distinct() } + + // TODO + override fun findMetadataPackageParts(packageFqName: String): List = TODO() } diff --git a/core/descriptors/src/org/jetbrains/kotlin/descriptors/PackagePartProvider.kt b/core/descriptors/src/org/jetbrains/kotlin/descriptors/PackagePartProvider.kt index 7265d8452ac..a28671fac77 100644 --- a/core/descriptors/src/org/jetbrains/kotlin/descriptors/PackagePartProvider.kt +++ b/core/descriptors/src/org/jetbrains/kotlin/descriptors/PackagePartProvider.kt @@ -25,7 +25,14 @@ interface PackagePartProvider { */ fun findPackageParts(packageFqName: String): List + /** + * @return simple names of .kotlin_metadata files that store data for top level declarations in the package with the given FQ name + */ + fun findMetadataPackageParts(packageFqName: String): List + object Empty : PackagePartProvider { override fun findPackageParts(packageFqName: String): List = emptyList() + + override fun findMetadataPackageParts(packageFqName: String): List = emptyList() } } diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/AbstractDeserializedPackageFragmentProvider.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/AbstractDeserializedPackageFragmentProvider.kt new file mode 100644 index 00000000000..4929388e278 --- /dev/null +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/AbstractDeserializedPackageFragmentProvider.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.serialization.deserialization + +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor +import org.jetbrains.kotlin.descriptors.PackageFragmentProvider +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.storage.StorageManager +import org.jetbrains.kotlin.utils.singletonOrEmptyList + +abstract class AbstractDeserializedPackageFragmentProvider( + protected val storageManager: StorageManager, + protected val finder: KotlinMetadataFinder, + protected val moduleDescriptor: ModuleDescriptor +) : PackageFragmentProvider { + protected lateinit var components: DeserializationComponents + + private val fragments = storageManager.createMemoizedFunctionWithNullableValues { fqName -> + findPackage(fqName)?.apply { + components = this@AbstractDeserializedPackageFragmentProvider.components + } + } + + protected abstract fun findPackage(fqName: FqName): DeserializedPackageFragment? + + override fun getPackageFragments(fqName: FqName): List = fragments(fqName).singletonOrEmptyList() + + override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection = emptySet() +} diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/DeserializedPackageFragment.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/DeserializedPackageFragment.kt index d5511462266..8cb441b7eb5 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/DeserializedPackageFragment.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/DeserializedPackageFragment.kt @@ -20,9 +20,9 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name -import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope +import org.jetbrains.kotlin.resolve.scopes.MemberScope +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedMemberScope import org.jetbrains.kotlin.storage.StorageManager -import org.jetbrains.kotlin.storage.getValue import javax.inject.Inject abstract class DeserializedPackageFragment( @@ -34,17 +34,16 @@ abstract class DeserializedPackageFragment( @set:Inject lateinit var components: DeserializationComponents - private val deserializedMemberScope by storageManager.createLazyValue { - computeMemberScope() - } + private val memberScope = storageManager.createLazyValue { computeMemberScope() } abstract val classDataFinder: ClassDataFinder - protected abstract fun computeMemberScope(): DeserializedPackageMemberScope + protected abstract fun computeMemberScope(): MemberScope - override fun getMemberScope() = deserializedMemberScope + override fun getMemberScope() = memberScope() - internal fun hasTopLevelClass(name: Name): Boolean { - return name in getMemberScope().classNames + open fun hasTopLevelClass(name: Name): Boolean { + val scope = getMemberScope() + return scope is DeserializedMemberScope && name in scope.classNames } } diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/KotlinMetadataFinder.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/KotlinMetadataFinder.kt index d7114348729..b429cd40a88 100644 --- a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/KotlinMetadataFinder.kt +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/KotlinMetadataFinder.kt @@ -16,10 +16,17 @@ package org.jetbrains.kotlin.serialization.deserialization +import org.jetbrains.kotlin.name.ClassId import org.jetbrains.kotlin.name.FqName import java.io.InputStream interface KotlinMetadataFinder { + /** + * @return an [InputStream] which should be used to load the .kotlin_metadata file for class with the given [classId]. + * [classId] identifies either a real top level class, or a package part (e.g. it can be "foo/bar/_1Kt") + */ + fun findMetadata(classId: ClassId): InputStream? + /** * @return an [InputStream] which should be used to load the .kotlin_builtins file for package with the given [packageFqName]. */ diff --git a/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt new file mode 100644 index 00000000000..4a74b9506c6 --- /dev/null +++ b/core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt @@ -0,0 +1,135 @@ +/* + * Copyright 2010-2016 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jetbrains.kotlin.serialization.deserialization + +import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol +import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion +import org.jetbrains.kotlin.descriptors.ModuleDescriptor +import org.jetbrains.kotlin.descriptors.PackagePartProvider +import org.jetbrains.kotlin.descriptors.SourceElement +import org.jetbrains.kotlin.incremental.components.LookupTracker +import org.jetbrains.kotlin.name.ClassId +import org.jetbrains.kotlin.name.FqName +import org.jetbrains.kotlin.name.Name +import org.jetbrains.kotlin.resolve.scopes.ChainedMemberScope +import org.jetbrains.kotlin.resolve.scopes.MemberScope +import org.jetbrains.kotlin.serialization.ClassData +import org.jetbrains.kotlin.serialization.ClassDataWithSource +import org.jetbrains.kotlin.serialization.ProtoBuf +import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf +import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope +import org.jetbrains.kotlin.storage.StorageManager +import java.io.InputStream + +class MetadataPackageFragmentProvider( + storageManager: StorageManager, + finder: KotlinMetadataFinder, + moduleDescriptor: ModuleDescriptor, + notFoundClasses: NotFoundClasses, + private val packagePartProvider: PackagePartProvider +) : AbstractDeserializedPackageFragmentProvider(storageManager, finder, moduleDescriptor) { + init { + components = DeserializationComponents( + storageManager, + moduleDescriptor, + DeserializationConfiguration.Default, // TODO + DeserializedClassDataFinder(this), + AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol), + this, + LocalClassifierTypeSettings.Default, + ErrorReporter.DO_NOTHING, + LookupTracker.Companion.DO_NOTHING, + FlexibleTypeDeserializer.ThrowException, + emptyList(), + notFoundClasses, AdditionalClassPartsProvider.None, PlatformDependentDeclarationFilter.All + ) + } + + override fun findPackage(fqName: FqName): DeserializedPackageFragment? = + MetadataPackageFragment(fqName, storageManager, moduleDescriptor, packagePartProvider, finder) +} + +class MetadataPackageFragment( + fqName: FqName, + storageManager: StorageManager, + module: ModuleDescriptor, + private val packagePartProvider: PackagePartProvider, + private val finder: KotlinMetadataFinder +) : DeserializedPackageFragment(fqName, storageManager, module) { + override val classDataFinder = ClassDataFinder { classId -> + val topLevelClassId = generateSequence(classId) { classId -> if (classId.isNestedClass) classId.outerClassId else null }.last() + val stream = finder.findMetadata(topLevelClassId) ?: return@ClassDataFinder null + val (message, nameResolver) = readProto(stream) + message.class_List.firstOrNull { classProto -> + nameResolver.getClassId(classProto.fqName) == classId + }?.let { classProto -> + ClassDataWithSource(ClassData(nameResolver, classProto), SourceElement.NO_SOURCE) + } + } + + override fun computeMemberScope(): MemberScope { + // For each .kotlin_metadata file which represents a package part, add a separate deserialized scope + // with top level callables and type aliases (but no classes) only from that part + val packageParts = packagePartProvider.findMetadataPackageParts(fqName.asString()) + val scopes = arrayListOf() + for (partName in packageParts) { + val stream = finder.findMetadata(ClassId(fqName, Name.identifier(partName))) ?: continue + val (proto, nameResolver) = readProto(stream) + + scopes.add(DeserializedPackageMemberScope( + this, proto.`package`, nameResolver, containerSource = null, components = components, classNames = { emptyList() } + )) + } + + // Also add the deserialized scope that can load all classes from this package + scopes.add(object : DeserializedPackageMemberScope( + this, ProtoBuf.Package.getDefaultInstance(), + NameResolverImpl(ProtoBuf.StringTable.getDefaultInstance(), ProtoBuf.QualifiedNameTable.getDefaultInstance()), + containerSource = null, components = components, classNames = { emptyList() } + ) { + override fun hasClass(name: Name): Boolean = hasTopLevelClass(name) + }) + + return ChainedMemberScope.create("Metadata scope", scopes) + } + + override fun hasTopLevelClass(name: Name): Boolean { + // TODO: check if the corresponding file exists + return true + } + + private fun readProto(stream: InputStream): Pair { + val version = BuiltInsBinaryVersion.readFrom(stream) + + if (!version.isCompatible()) { + // TODO: report a proper diagnostic + throw UnsupportedOperationException( + "Kotlin metadata definition format version is not supported: " + + "expected ${BuiltInsBinaryVersion.INSTANCE}, actual $version. " + + "Please update Kotlin" + ) + } + + val message = BuiltInsProtoBuf.BuiltIns.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry) + val nameResolver = NameResolverImpl(message.strings, message.qualifiedNames) + return Pair(message, nameResolver) + } + + companion object { + val METADATA_FILE_EXTENSION = ".kotlin_metadata" + } +} diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IDEPackagePartProvider.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IDEPackagePartProvider.kt index ec4d7851217..92fbfc4e067 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IDEPackagePartProvider.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IDEPackagePartProvider.kt @@ -18,14 +18,17 @@ package org.jetbrains.kotlin.idea.caches.resolve import com.intellij.psi.search.GlobalSearchScope import com.intellij.util.indexing.FileBasedIndex -import org.jetbrains.kotlin.idea.vfilefinder.KotlinModuleMappingIndex import org.jetbrains.kotlin.descriptors.PackagePartProvider +import org.jetbrains.kotlin.idea.vfilefinder.KotlinModuleMappingIndex import org.jetbrains.kotlin.load.kotlin.PackageParts class IDEPackagePartProvider(val scope: GlobalSearchScope) : PackagePartProvider { + override fun findPackageParts(packageFqName: String): List = + getPackageParts(packageFqName).flatMap(PackageParts::parts).distinct() - override fun findPackageParts(packageFqName: String): List { - val values: MutableList = FileBasedIndex.getInstance().getValues(KotlinModuleMappingIndex.KEY, packageFqName, scope) - return values.flatMap { it.parts }.distinct() - } -} \ No newline at end of file + override fun findMetadataPackageParts(packageFqName: String): List = + getPackageParts(packageFqName).flatMap(PackageParts::metadataParts).distinct() + + private fun getPackageParts(packageFqName: String): MutableList = + FileBasedIndex.getInstance().getValues(KotlinModuleMappingIndex.KEY, packageFqName, scope) +} diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt index f8d545a1656..35f55a23b59 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt @@ -116,6 +116,9 @@ class DirectoryBasedClassFinder( return null } + // TODO + override fun findMetadata(classId: ClassId): InputStream? = null + // TODO: load built-ins from packageDirectory? override fun findBuiltInsData(packageFqName: FqName): InputStream? = null } diff --git a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt index ef5f7f7dd6e..b0a36acd387 100644 --- a/idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt +++ b/idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt @@ -56,6 +56,9 @@ class JsIDEVirtualFileFinder(private val scope: GlobalSearchScope) : JsVirtualFi } class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFileKotlinClassFinder(), JvmVirtualFileFinder { + // TODO + override fun findMetadata(classId: ClassId): InputStream? = null + // TODO: load built-ins metadata from scope override fun findBuiltInsData(packageFqName: FqName): InputStream? = null diff --git a/idea/src/META-INF/plugin.xml b/idea/src/META-INF/plugin.xml index ad00f4b8d3b..61a575364bb 100644 --- a/idea/src/META-INF/plugin.xml +++ b/idea/src/META-INF/plugin.xml @@ -227,6 +227,9 @@ + +