Load definitions of symbols from .kotlin_metadata files

Extract AbstractDeserializedPackageFragmentProvider out of
JvmBuiltInsPackageFragmentProvider and implement it a little bit differently in
MetadataPackageFragmentProvider. The main difference is in how the package
fragment scope is constructed: for built-ins, it's just a single scope that
loads everything from one protobuf message. For metadata, package scope can
consist of many files, some of which store information about classes and others
are similar to package parts on JVM, so a ChainedMemberScope instance is
created.

Introduce a bunch of interfaces/methods to deliver the needed behavior to the
'deserialization' module which is not JVM-specific and does not depend on the
compiler code: MetadataFinderFactory,
PackagePartProvider#findMetadataPackageParts, KotlinMetadataFinder#findMetadata.
Note that these declarations are currently only implemented in the compiler; no
metadata package parts/fragments will be found in IDE or reflection

compiler/builtins-serializer/src/org/jetbrains/kotlin/serialization/MetadataSerializer.kt

@@ -23,6 +23,7 @@ import org.jetbrains.kotlin.cli.common.CLIConfigurationKeys
 import org.jetbrains.kotlin.cli.common.messages.AnalyzerWithCompilerReport
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageLocation
 import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity
+import org.jetbrains.kotlin.cli.jvm.compiler.JvmPackagePartProvider
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.codegen.JvmCodegenUtil
 import org.jetbrains.kotlin.codegen.serializeToByteArray
@@ -41,6 +42,7 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsProtoBuf
 import org.jetbrains.kotlin.serialization.builtins.BuiltInsSerializerExtension
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment.Companion.METADATA_FILE_EXTENSION
 import org.jetbrains.kotlin.serialization.jvm.JvmPackageTable
 import java.io.ByteArrayOutputStream
 import java.io.DataOutputStream
@@ -63,7 +65,9 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
 
         val analyzer = AnalyzerWithCompilerReport(messageCollector)
         analyzer.analyzeAndReport(files, object : AnalyzerWithCompilerReport.Analyzer {
-            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns)
+            override fun analyze(): AnalysisResult = DefaultAnalyzerFacade.analyzeFiles(files, moduleName, dependOnOldBuiltIns) {
+                _, content -> JvmPackagePartProvider(environment, content.moduleContentScope)
+            }
         })
 
         if (analyzer.hasErrors()) return
@@ -193,8 +197,4 @@ open class MetadataSerializer(private val dependOnOldBuiltIns: Boolean) {
             destFile.writeBytes(stream.toByteArray())
         }
     }
-
-    companion object {
-        private val METADATA_FILE_EXTENSION = ".kotlin_metadata"
-    }
 }

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt

@@ -25,6 +25,7 @@ import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragment
 import org.jetbrains.kotlin.utils.addToStdlib.check
 import java.io.InputStream
 
@@ -35,6 +36,12 @@ class JvmCliVirtualFileFinder(
     override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? =
             findBinaryClass(classId, classId.relativeClassName.asString().replace('.', '$') + ".class")
 
+    override fun findMetadata(classId: ClassId): InputStream? {
+        assert(!classId.isNestedClass) { "Nested classes are not supported here: $classId" }
+
+        return findBinaryClass(classId, classId.shortClassName.asString() + MetadataPackageFragment.METADATA_FILE_EXTENSION)?.inputStream
+    }
+
     override fun findBuiltInsData(packageFqName: FqName): InputStream? {
         // "<builtins-metadata>" is just a made-up name
         // JvmDependenciesIndex requires the ClassId of the class which we're searching for, to cache the last request+result

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmPackagePartProvider.kt

@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.cli.jvm.config.JvmClasspathRoot
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
 import org.jetbrains.kotlin.descriptors.PackagePartProvider
 import org.jetbrains.kotlin.load.kotlin.ModuleMapping
+import org.jetbrains.kotlin.load.kotlin.PackageParts
 import java.io.EOFException
 
 class JvmPackagePartProvider(
@@ -38,11 +39,16 @@ class JvmPackagePartProvider(
 
     private val loadedModules: MutableList<ModuleMapping> = SmartList()
 
-    @Synchronized
-    override fun findPackageParts(packageFqName: String): List<String> {
-        processNotLoadedRelevantRoots(packageFqName)
+    override fun findPackageParts(packageFqName: String): List<String> =
+            getPackageParts(packageFqName).flatMap(PackageParts::parts).distinct()
 
-        return loadedModules.flatMap { it.findPackageParts(packageFqName)?.parts ?: emptySet<String>() }.distinct()
+    override fun findMetadataPackageParts(packageFqName: String): List<String> =
+            getPackageParts(packageFqName).flatMap(PackageParts::metadataParts).distinct()
+
+    @Synchronized
+    private fun getPackageParts(packageFqName: String): List<PackageParts> {
+        processNotLoadedRelevantRoots(packageFqName)
+        return loadedModules.mapNotNull { it.findPackageParts(packageFqName) }
     }
 
     private fun processNotLoadedRelevantRoots(packageFqName: String) {

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/KotlinCoreEnvironment.kt

@@ -86,6 +86,7 @@ import org.jetbrains.kotlin.extensions.StorageComponentContainerContributor
 import org.jetbrains.kotlin.idea.KotlinFileType
 import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinderFactory
 import org.jetbrains.kotlin.load.kotlin.KotlinBinaryClassCache
+import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
 import org.jetbrains.kotlin.load.kotlin.ModuleVisibilityManager
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.isValidJavaFqName
@@ -160,7 +161,9 @@ class KotlinCoreEnvironment private constructor(
         (ServiceManager.getService(project, CoreJavaFileManager::class.java)
                 as KotlinCliJavaFileManagerImpl).initIndex(rootsIndex)
 
-        project.registerService(JvmVirtualFileFinderFactory::class.java, JvmCliVirtualFileFinderFactory(rootsIndex))
+        val finderFactory = JvmCliVirtualFileFinderFactory(rootsIndex)
+        project.registerService(MetadataFinderFactory::class.java, finderFactory)
+        project.registerService(JvmVirtualFileFinderFactory::class.java, finderFactory)
 
         ExpressionCodegenExtension.registerExtensionPoint(project)
         ClassBuilderInterceptorExtension.registerExtensionPoint(project)

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/JvmVirtualFileFinderFactory.kt

@@ -20,7 +20,7 @@ import com.intellij.openapi.components.ServiceManager
 import com.intellij.openapi.project.Project
 import com.intellij.psi.search.GlobalSearchScope
 
-interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory {
+interface JvmVirtualFileFinderFactory : VirtualFileFinderFactory, MetadataFinderFactory {
     override fun create(scope: GlobalSearchScope): JvmVirtualFileFinder
 
     object SERVICE {

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt

@@ -39,6 +39,9 @@ internal class IncrementalPackagePartProvider private constructor(
                 parent.findPackageParts(packageFqName)).distinct()
     }
 
+    // TODO
+    override fun findMetadataPackageParts(packageFqName: String): List<String> = TODO()
+
     companion object {
         @JvmStatic
         fun create(

compiler/frontend/src/org/jetbrains/kotlin/analyzer/common/DefaultAnalyzerFacade.kt

@@ -16,6 +16,7 @@
 
 package org.jetbrains.kotlin.analyzer.common
 
+import com.intellij.openapi.components.ServiceManager
 import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.analyzer.*
 import org.jetbrains.kotlin.config.ApiVersion
@@ -30,9 +31,11 @@ import org.jetbrains.kotlin.context.LazyResolveToken
 import org.jetbrains.kotlin.context.ModuleContext
 import org.jetbrains.kotlin.context.ProjectContext
 import org.jetbrains.kotlin.descriptors.PackagePartProvider
+import org.jetbrains.kotlin.descriptors.impl.CompositePackageFragmentProvider
 import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.frontend.di.configureModule
 import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.kotlin.MetadataFinderFactory
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.*
@@ -40,6 +43,7 @@ import org.jetbrains.kotlin.resolve.lazy.FileScopeProviderImpl
 import org.jetbrains.kotlin.resolve.lazy.ResolveSession
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactoryService
+import org.jetbrains.kotlin.serialization.deserialization.MetadataPackageFragmentProvider
 
 /**
  * A facade that is used to analyze platform independent modules in multi-platform projects.
@@ -97,7 +101,12 @@ object DefaultAnalyzerFacade : AnalyzerFacade<PlatformAnalysisParameters>() {
                 moduleContext, trace, declarationProviderFactory, moduleContentScope, targetEnvironment, packagePartProvider
         )
 
-        return ResolverForModule(container.get<ResolveSession>().packageFragmentProvider, container)
+        val packageFragmentProviders = listOf(
+                container.get<ResolveSession>().packageFragmentProvider,
+                container.get<MetadataPackageFragmentProvider>()
+        )
+
+        return ResolverForModule(CompositePackageFragmentProvider(packageFragmentProviders), container)
     }
 
     private fun createContainerToResolveCommonCode(
@@ -119,6 +128,11 @@ object DefaultAnalyzerFacade : AnalyzerFacade<PlatformAnalysisParameters>() {
         useImpl<CompilerDeserializationConfiguration>()
         useInstance(packagePartProvider)
         useInstance(declarationProviderFactory)
+        useImpl<MetadataPackageFragmentProvider>()
+
+        val metadataFinderFactory = ServiceManager.getService(moduleContext.project, MetadataFinderFactory::class.java)
+                                    ?: error("No MetadataFinderFactory in project")
+        useInstance(metadataFinderFactory.create(moduleContentScope))
 
         targetEnvironment.configure(this)
         useImpl<LazyResolveToken>()

compiler/frontend/src/org/jetbrains/kotlin/load/kotlin/MetadataFinderFactory.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.load.kotlin
+
+import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.serialization.deserialization.KotlinMetadataFinder
+
+interface MetadataFinderFactory {
+    fun create(scope: GlobalSearchScope): KotlinMetadataFinder
+}