Load definitions of symbols from .kotlin_metadata files

Extract AbstractDeserializedPackageFragmentProvider out of
JvmBuiltInsPackageFragmentProvider and implement it a little bit differently in
MetadataPackageFragmentProvider. The main difference is in how the package
fragment scope is constructed: for built-ins, it's just a single scope that
loads everything from one protobuf message. For metadata, package scope can
consist of many files, some of which store information about classes and others
are similar to package parts on JVM, so a ChainedMemberScope instance is
created.

Introduce a bunch of interfaces/methods to deliver the needed behavior to the
'deserialization' module which is not JVM-specific and does not depend on the
compiler code: MetadataFinderFactory,
PackagePartProvider#findMetadataPackageParts, KotlinMetadataFinder#findMetadata.
Note that these declarations are currently only implemented in the compiler; no
metadata package parts/fragments will be found in IDE or reflection

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt

@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.jvm.compiler.JvmPackagePartProvider
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.codegen.JvmCodegenUtil
 import org.jetbrains.kotlin.codegen.serializeToByteArray
@@ -41,6 +42,7 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment.Companion.METADATA_FILE_EXTENSION
 import org.jetbrains.kotlin.serialization.jvm.JvmPackageTable
 import java.io.ByteArrayOutputStream
 import java.io.DataOutputStream
@@ -63,7 +65,9 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
 
         val analyzer = AnalyzerWithCompilerReport(messageCollector)
         analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer {
-            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns)
+            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) {
+                _, content -> JvmPackagePartProvider(environment, content.moduleContentScope)
+            }
         })
 
         if (analyzer.hasErrors()) return
@@ -193,8 +197,4 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
             destFile.writeBytes(stream.toByteArray())
         }
     }
-
-    companion object {
-        private val METADATA_FILE_EXTENSION = ".kotlin_metadata"
-    }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt

@@ -25,6 +25,7 @@ import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment
 import org.jetbrains.kotlin.utils.addToStdlib.check
 import java.io.InputStream
 
@@ -35,6 +36,12 @@ class JvmCliVirtualFileFinder(
     override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? =
             findBinaryClass(classId, classId.relativeClassName.asString().replace('.', '$') + ".class")
 
+    override fun findMetadata(classId: ClassId): InputStream? {
+        assert(!classId.isNestedClass) { "Nested classes are not supported here: $classId" }
+
+        return findBinaryClass(classId, classId.shortClassName.asString() + MetadataPackageFragment.METADATA_FILE_EXTENSION)?.inputStream
+    }
+
     override fun findBuiltInsData(packageFqName: FqName): InputStream? {
         // "<builtins-metadata>" is just a made-up name
         // JvmDependenciesIndex requires the ClassId of the class which we're searching for, to cache the last request+result

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt

@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.descriptors.PackagePartProvider
 import org.jetbrains.kotlin.load.kotlin.ModuleMapping
+import org.jetbrains.kotlin.load.kotlin.PackageParts
 import java.io.EOFException
 
 class JvmPackagePartProvider(
@@ -38,11 +39,16 @@ class JvmPackagePartProvider(
 
     private val loadedModules: MutableList<ModuleMapping> = SmartList()
 
-    @Synchronized
+    override fun findPackageParts(packageFqName: String): List<String> =
-    override fun findPackageParts(packageFqName: String): List<String> {
+            getPackageParts(packageFqName).flatMap(PackageParts::parts).distinct()
-        processNotLoadedRelevantRoots(packageFqName)
 
-        return loadedModules.flatMap { it.findPackageParts(packageFqName)?.parts ?: emptySet<String>() }.distinct()
+    override fun findMetadataPackageParts(packageFqName: String): List<String> =
+            getPackageParts(packageFqName).flatMap(PackageParts::metadataParts).distinct()
+
+    @Synchronized
+    private fun getPackageParts(packageFqName: String): List<PackageParts> {
+        processNotLoadedRelevantRoots(packageFqName)
+        return loadedModules.mapNotNull { it.findPackageParts(packageFqName) }
     }
 
     private fun processNotLoadedRelevantRoots(packageFqName: String) {

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -86,6 +86,7 @@ import org.jetbrains.kotlin.extensions.StorageComponentContainerContributor
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory
 import org.jetbrains.kotlin.load.kotlin.KotlinBinaryClassCache
+import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
 import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.isValidJavaFqName
@@ -160,7 +161,9 @@ class KotlinCoreEnvironment private constructor(
         (ServiceManager.getService(project, CoreJavaFileManager::class.java)
                 as KotlinCliJavaFileManagerImpl).initIndex(rootsIndex)
 
-        project.registerService(JvmVirtualFileFinderFactory::class.java, JvmCliVirtualFileFinderFactory(rootsIndex))
+        val finderFactory = JvmCliVirtualFileFinderFactory(rootsIndex)
+        project.registerService(MetadataFinderFactory::class.java, finderFactory)
+        project.registerService(JvmVirtualFileFinderFactory::class.java, finderFactory)
 
         ExpressionCodegenExtension.registerExtensionPoint(project)
         ClassBuilderInterceptorExtension.registerExtensionPoint(project)

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt

@@ -20,7 +20,7 @@ import com.intellij.openapi.components.ServiceManager
 import com.intellij.openapi.project.Project
 import com.intellij.psi.search.GlobalSearchScope
 
-interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory {
+interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory, MetadataFinderFactory {
     override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder
 
     object SERVICE {

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt

@@ -39,6 +39,9 @@ internal class IncrementalPackagePartProvider private constructor(
                 parent.findPackageParts(packageFqName)).distinct()
     }
 
+    // TODO
+    override fun findMetadataPackageParts(packageFqName: String): List<String> = TODO()
+
     companion object {
         @JvmStatic
         fun create(

compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.analyzer.common
 
+import com.intellij.openapi.components.ServiceManager
 import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.analyzer.*
 import org.jetbrains.kotlin.config.ApiVersion
@@ -30,9 +31,11 @@ import org.jetbrains.kotlin.context.LazyResolveToken
 import org.jetbrains.kotlin.context.ModuleContext
 import org.jetbrains.kotlin.context.ProjectContext
 import org.jetbrains.kotlin.descriptors.PackagePartProvider
+import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.frontend.di.configureModule
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.*
@@ -40,6 +43,7 @@ import org.jetbrains.kotlin.resolve.lazy.FileScopeProviderImpl
 import org.jetbrains.kotlin.resolve.lazy.ResolveSession
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragmentProvider
 
 /**
  * A facade that is used to analyze platform independent modules in multi-platform projects.
@@ -97,7 +101,12 @@ object DefaultAnalyzerFacade : AnalyzerFacade<PlatformAnalysisParameters>() {
                 moduleContext, trace, declarationProviderFactory, moduleContentScope, targetEnvironment, packagePartProvider
         )
 
-        return ResolverForModule(container.get<ResolveSession>().packageFragmentProvider, container)
+        val packageFragmentProviders = listOf(
+                container.get<ResolveSession>().packageFragmentProvider,
+                container.get<MetadataPackageFragmentProvider>()
+        )
+
+        return ResolverForModule(CompositePackageFragmentProvider(packageFragmentProviders), container)
     }
 
     private fun createContainerToResolveCommonCode(
@@ -119,6 +128,11 @@ object DefaultAnalyzerFacade : AnalyzerFacade<PlatformAnalysisParameters>() {
         useImpl<CompilerDeserializationConfiguration>()
         useInstance(packagePartProvider)
         useInstance(declarationProviderFactory)
+        useImpl<MetadataPackageFragmentProvider>()
+
+        val metadataFinderFactory = ServiceManager.getService(moduleContext.project, MetadataFinderFactory::class.java)
+                                    ?: error("No MetadataFinderFactory in project")
+        useInstance(metadataFinderFactory.create(moduleContentScope))
 
         targetEnvironment.configure(this)
         useImpl<LazyResolveToken>()

compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.load.kotlin
+
+import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.serialization.deserialization.KotlinMetadataFinder
+
+interface MetadataFinderFactory {
+    fun create(scope: GlobalSearchScope): KotlinMetadataFinder
+}

core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt

@@ -18,15 +18,11 @@ package org.jetbrains.kotlin.builtins
 
 import org.jetbrains.kotlin.builtins.functions.BuiltInFictitiousFunctionClassFactory
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
-import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
 import org.jetbrains.kotlin.incremental.components.LookupTracker
 import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.serialization.deserialization.*
 import org.jetbrains.kotlin.storage.StorageManager
-import org.jetbrains.kotlin.utils.singletonOrEmptyList
 
 class JvmBuiltInsPackageFragmentProvider(
         storageManager: StorageManager,
@@ -35,22 +31,12 @@ class JvmBuiltInsPackageFragmentProvider(
         notFoundClasses: NotFoundClasses,
         additionalClassPartsProvider: AdditionalClassPartsProvider,
         platformDependentDeclarationFilter: PlatformDependentDeclarationFilter
-) : PackageFragmentProvider {
+) : AbstractDeserializedPackageFragmentProvider(storageManager, finder, moduleDescriptor) {
-    private lateinit var components: DeserializationComponents
-
-    private val fragments = storageManager.createMemoizedFunctionWithNullableValues<FqName, PackageFragmentDescriptor> { fqName ->
-        finder.findBuiltInsData(fqName)?.let { inputStream ->
-            BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor, inputStream).apply {
-                components = this@JvmBuiltInsPackageFragmentProvider.components
-            }
-        }
-    }
-
     init {
         components = DeserializationComponents(
                 storageManager,
                 moduleDescriptor,
-                DeserializationConfiguration.Default,
+                DeserializationConfiguration.Default, // TODO
                 DeserializedClassDataFinder(this),
                 AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol),
                 this,
@@ -64,10 +50,10 @@ class JvmBuiltInsPackageFragmentProvider(
                 ),
                 notFoundClasses, additionalClassPartsProvider, platformDependentDeclarationFilter
         )
-
     }
 
-    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> = fragments(fqName).singletonOrEmptyList()
+    override fun findPackage(fqName: FqName): DeserializedPackageFragment? =
-
+            finder.findBuiltInsData(fqName)?.let { inputStream ->
-    override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection<FqName> = emptySet()
+                BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor, inputStream)
+            }
 }

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/ModuleMapping.kt

@@ -34,24 +34,20 @@ class ModuleMapping private constructor(val packageFqName2Parts: Map<String, Pac
         @JvmField
         val EMPTY: ModuleMapping = ModuleMapping(emptyMap(), "EMPTY")
 
-        fun create(proto: ByteArray?, debugName: String?): ModuleMapping {
+        fun create(bytes: ByteArray?, debugName: String?): ModuleMapping {
-            if (proto == null) {
+            if (bytes == null) {
                 return EMPTY
             }
 
-            val stream = DataInputStream(ByteArrayInputStream(proto))
+            val stream = DataInputStream(ByteArrayInputStream(bytes))
             val version = JvmMetadataVersion(*IntArray(stream.readInt()) { stream.readInt() })
 
             if (version.isCompatible()) {
                 val parseFrom = JvmPackageTable.PackageTable.parseFrom(stream)
                 if (parseFrom != null) {
-                    val packageFqNameParts = hashMapOf<String, PackageParts>()
+                    val packageFqNameParts = hashMapOf<String, PackageParts>().apply {
-                    parseFrom.packagePartsList.forEach {
+                        addParts(this, parseFrom.packagePartsList, PackageParts::parts)
-                        val packageParts = PackageParts(it.packageFqName)
+                        addParts(this, parseFrom.metadataPartsList, PackageParts::metadataParts)
-                        packageFqNameParts.put(it.packageFqName, packageParts)
-                        it.classNameList.forEach {
-                            packageParts.parts.add(it)
-                        }
                     }
                     return ModuleMapping(packageFqNameParts, debugName ?: "<unknown>")
                 }
@@ -62,6 +58,19 @@ class ModuleMapping private constructor(val packageFqName2Parts: Map<String, Pac
 
             return EMPTY
         }
+
+        private inline fun addParts(
+                result: MutableMap<String, PackageParts>,
+                partsList: List<JvmPackageTable.PackageParts>,
+                whichParts: (PackageParts) -> MutableSet<String>
+        ) {
+            for (proto in partsList) {
+                PackageParts(proto.packageFqName).apply {
+                    result.put(proto.packageFqName, this)
+                    whichParts(this).addAll(proto.classNameList)
+                }
+            }
+        }
     }
 }
 

core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt

@@ -36,6 +36,9 @@ class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinCla
         return findKotlinClass(javaClass.fqName?.asString() ?: return null)
     }
 
+    // TODO
+    override fun findMetadata(classId: ClassId): InputStream? = null
+
     // TODO: load built-ins from classLoader
     override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }

core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/RuntimePackagePartProvider.kt

@@ -40,4 +40,7 @@ class RuntimePackagePartProvider(private val classLoader: ClassLoader) : Package
     override fun findPackageParts(packageFqName: String): List<String> {
         return module2Mapping.values.mapNotNull { it.findPackageParts(packageFqName) }.flatMap { it.parts }.distinct()
     }
+
+    // TODO
+    override fun findMetadataPackageParts(packageFqName: String): List<String> = TODO()
 }

core/descriptors/src/org/jetbrains/kotlin/descriptors/PackagePartProvider.kt

@@ -25,7 +25,14 @@ interface PackagePartProvider {
      */
     fun findPackageParts(packageFqName: String): List<String>
 
+    /**
+     * @return simple names of .kotlin_metadata files that store data for top level declarations in the package with the given FQ name
+     */
+    fun findMetadataPackageParts(packageFqName: String): List<String>
+
     object Empty : PackagePartProvider {
         override fun findPackageParts(packageFqName: String): List<String> = emptyList()
+
+        override fun findMetadataPackageParts(packageFqName: String): List<String> = emptyList()
     }
 }

core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/AbstractDeserializedPackageFragmentProvider.kt

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.serialization.deserialization
+
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.utils.singletonOrEmptyList
+
+abstract class AbstractDeserializedPackageFragmentProvider(
+        protected val storageManager: StorageManager,
+        protected val finder: KotlinMetadataFinder,
+        protected val moduleDescriptor: ModuleDescriptor
+) : PackageFragmentProvider {
+    protected lateinit var components: DeserializationComponents
+
+    private val fragments = storageManager.createMemoizedFunctionWithNullableValues<FqName, PackageFragmentDescriptor> { fqName ->
+        findPackage(fqName)?.apply {
+            components = this@AbstractDeserializedPackageFragmentProvider.components
+        }
+    }
+
+    protected abstract fun findPackage(fqName: FqName): DeserializedPackageFragment?
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> = fragments(fqName).singletonOrEmptyList()
+
+    override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection<FqName> = emptySet()
+}

core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/DeserializedPackageFragment.kt

@@ -20,9 +20,9 @@ import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
+import org.jetbrains.kotlin.resolve.scopes.MemberScope
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedMemberScope
 import org.jetbrains.kotlin.storage.StorageManager
-import org.jetbrains.kotlin.storage.getValue
 import javax.inject.Inject
 
 abstract class DeserializedPackageFragment(
@@ -34,17 +34,16 @@ abstract class DeserializedPackageFragment(
     @set:Inject
     lateinit var components: DeserializationComponents
 
-    private val deserializedMemberScope by storageManager.createLazyValue {
+    private val memberScope = storageManager.createLazyValue { computeMemberScope() }
-        computeMemberScope()
-    }
 
     abstract val classDataFinder: ClassDataFinder
 
-    protected abstract fun computeMemberScope(): DeserializedPackageMemberScope
+    protected abstract fun computeMemberScope(): MemberScope
 
-    override fun getMemberScope() = deserializedMemberScope
+    override fun getMemberScope() = memberScope()
 
-    internal fun hasTopLevelClass(name: Name): Boolean {
+    open fun hasTopLevelClass(name: Name): Boolean {
-        return name in getMemberScope().classNames
+        val scope = getMemberScope()
+        return scope is DeserializedMemberScope && name in scope.classNames
     }
 }

core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/KotlinMetadataFinder.kt

@@ -16,10 +16,17 @@
 
 package org.jetbrains.kotlin.serialization.deserialization
 
+import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import java.io.InputStream
 
 interface KotlinMetadataFinder {
+    /**
+     * @return an [InputStream] which should be used to load the .kotlin_metadata file for class with the given [classId].
+     * [classId] identifies either a real top level class, or a package part (e.g. it can be "foo/bar/_1Kt")
+     */
+    fun findMetadata(classId: ClassId): InputStream?
+
     /**
      * @return an [InputStream] which should be used to load the .kotlin_builtins file for package with the given [packageFqName].
      */

core/deserialization/src/org/jetbrains/kotlin/serialization/deserialization/MetadataPackageFragmentProvider.kt

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.serialization.deserialization
+
+import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
+import org.jetbrains.kotlin.builtins.BuiltInsBinaryVersion
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackagePartProvider
+import org.jetbrains.kotlin.descriptors.SourceElement
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.resolve.scopes.ChainedMemberScope
+import org.jetbrains.kotlin.resolve.scopes.MemberScope
+import org.jetbrains.kotlin.serialization.ClassData
+import org.jetbrains.kotlin.serialization.ClassDataWithSource
+import org.jetbrains.kotlin.serialization.ProtoBuf
+import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
+import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
+import org.jetbrains.kotlin.storage.StorageManager
+import java.io.InputStream
+
+class MetadataPackageFragmentProvider(
+        storageManager: StorageManager,
+        finder: KotlinMetadataFinder,
+        moduleDescriptor: ModuleDescriptor,
+        notFoundClasses: NotFoundClasses,
+        private val packagePartProvider: PackagePartProvider
+) : AbstractDeserializedPackageFragmentProvider(storageManager, finder, moduleDescriptor) {
+    init {
+        components = DeserializationComponents(
+                storageManager,
+                moduleDescriptor,
+                DeserializationConfiguration.Default, // TODO
+                DeserializedClassDataFinder(this),
+                AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol),
+                this,
+                LocalClassifierTypeSettings.Default,
+                ErrorReporter.DO_NOTHING,
+                LookupTracker.Companion.DO_NOTHING,
+                FlexibleTypeDeserializer.ThrowException,
+                emptyList(),
+                notFoundClasses, AdditionalClassPartsProvider.None, PlatformDependentDeclarationFilter.All
+        )
+    }
+
+    override fun findPackage(fqName: FqName): DeserializedPackageFragment? =
+            MetadataPackageFragment(fqName, storageManager, moduleDescriptor, packagePartProvider, finder)
+}
+
+class MetadataPackageFragment(
+        fqName: FqName,
+        storageManager: StorageManager,
+        module: ModuleDescriptor,
+        private val packagePartProvider: PackagePartProvider,
+        private val finder: KotlinMetadataFinder
+) : DeserializedPackageFragment(fqName, storageManager, module) {
+    override val classDataFinder = ClassDataFinder { classId ->
+        val topLevelClassId = generateSequence(classId) { classId -> if (classId.isNestedClass) classId.outerClassId else null }.last()
+        val stream = finder.findMetadata(topLevelClassId) ?: return@ClassDataFinder null
+        val (message, nameResolver) = readProto(stream)
+        message.class_List.firstOrNull { classProto ->
+            nameResolver.getClassId(classProto.fqName) == classId
+        }?.let { classProto ->
+            ClassDataWithSource(ClassData(nameResolver, classProto), SourceElement.NO_SOURCE)
+        }
+    }
+
+    override fun computeMemberScope(): MemberScope {
+        // For each .kotlin_metadata file which represents a package part, add a separate deserialized scope
+        // with top level callables and type aliases (but no classes) only from that part
+        val packageParts = packagePartProvider.findMetadataPackageParts(fqName.asString())
+        val scopes = arrayListOf<DeserializedPackageMemberScope>()
+        for (partName in packageParts) {
+            val stream = finder.findMetadata(ClassId(fqName, Name.identifier(partName))) ?: continue
+            val (proto, nameResolver) = readProto(stream)
+
+            scopes.add(DeserializedPackageMemberScope(
+                    this, proto.`package`, nameResolver, containerSource = null, components = components, classNames = { emptyList() }
+            ))
+        }
+
+        // Also add the deserialized scope that can load all classes from this package
+        scopes.add(object : DeserializedPackageMemberScope(
+                this, ProtoBuf.Package.getDefaultInstance(),
+                NameResolverImpl(ProtoBuf.StringTable.getDefaultInstance(), ProtoBuf.QualifiedNameTable.getDefaultInstance()),
+                containerSource = null, components = components, classNames = { emptyList() }
+        ) {
+            override fun hasClass(name: Name): Boolean = hasTopLevelClass(name)
+        })
+
+        return ChainedMemberScope.create("Metadata scope", scopes)
+    }
+
+    override fun hasTopLevelClass(name: Name): Boolean {
+        // TODO: check if the corresponding file exists
+        return true
+    }
+
+    private fun readProto(stream: InputStream): Pair<BuiltInsProtoBuf.BuiltIns, NameResolverImpl> {
+        val version = BuiltInsBinaryVersion.readFrom(stream)
+
+        if (!version.isCompatible()) {
+            // TODO: report a proper diagnostic
+            throw UnsupportedOperationException(
+                    "Kotlin metadata definition format version is not supported: " +
+                    "expected ${BuiltInsBinaryVersion.INSTANCE}, actual $version. " +
+                    "Please update Kotlin"
+            )
+        }
+
+        val message = BuiltInsProtoBuf.BuiltIns.parseFrom(stream, BuiltInSerializerProtocol.extensionRegistry)
+        val nameResolver = NameResolverImpl(message.strings, message.qualifiedNames)
+        return Pair(message, nameResolver)
+    }
+
+    companion object {
+        val METADATA_FILE_EXTENSION = ".kotlin_metadata"
+    }
+}

idea/idea-analysis/src/org/jetbrains/kotlin/idea/caches/resolve/IDEPackagePartProvider.kt

@@ -18,14 +18,17 @@ package org.jetbrains.kotlin.idea.caches.resolve
 
 import com.intellij.psi.search.GlobalSearchScope
 import com.intellij.util.indexing.FileBasedIndex
-import org.jetbrains.kotlin.idea.vfilefinder.KotlinModuleMappingIndex
 import org.jetbrains.kotlin.descriptors.PackagePartProvider
+import org.jetbrains.kotlin.idea.vfilefinder.KotlinModuleMappingIndex
 import org.jetbrains.kotlin.load.kotlin.PackageParts
 
 class IDEPackagePartProvider(val scope: GlobalSearchScope) : PackagePartProvider {
+    override fun findPackageParts(packageFqName: String): List<String> =
+            getPackageParts(packageFqName).flatMap(PackageParts::parts).distinct()
 
-    override fun findPackageParts(packageFqName: String): List<String> {
+    override fun findMetadataPackageParts(packageFqName: String): List<String> =
-        val values: MutableList<PackageParts> = FileBasedIndex.getInstance().getValues(KotlinModuleMappingIndex.KEY, packageFqName, scope)
+            getPackageParts(packageFqName).flatMap(PackageParts::metadataParts).distinct()
-        return values.flatMap { it.parts }.distinct()
+
-    }
+    private fun getPackageParts(packageFqName: String): MutableList<PackageParts> =
-}
+            FileBasedIndex.getInstance().getValues(KotlinModuleMappingIndex.KEY, packageFqName, scope)
+}

idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt

@@ -116,6 +116,9 @@ class DirectoryBasedClassFinder(
         return null
     }
 
+    // TODO
+    override fun findMetadata(classId: ClassId): InputStream? = null
+
     // TODO: load built-ins from packageDirectory?
     override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }

idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt

@@ -56,6 +56,9 @@ class JsIDEVirtualFileFinder(private val scope: GlobalSearchScope) : JsVirtualFi
 }
 
 class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFileKotlinClassFinder(), JvmVirtualFileFinder {
+    // TODO
+    override fun findMetadata(classId: ClassId): InputStream? = null
+
     // TODO: load built-ins metadata from scope
     override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 

idea/src/META-INF/plugin.xml

@@ -227,6 +227,9 @@
     <projectService serviceInterface="org.jetbrains.kotlin.caches.resolve.KotlinCacheService"
                     serviceImplementation="org.jetbrains.kotlin.idea.caches.resolve.KotlinCacheServiceImpl"/>
 
+    <projectService serviceInterface="org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory"
+                    serviceImplementation="org.jetbrains.kotlin.idea.vfilefinder.JvmIDEVirtualFileFinderFactory"/>
+
     <projectService serviceInterface="org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory"
                     serviceImplementation="org.jetbrains.kotlin.idea.vfilefinder.JvmIDEVirtualFileFinderFactory"/>