Load built-ins from module dependencies in JVM compiler

Introduce a new method KotlinClassFinder#findBuiltInsData, which is only
implemented correctly in the JvmCliVirtualFileFinder because it's only used in
the compiler code at the moment.

Introduce JvmBuiltInsPackageFragmentProvider, the purpose of which is to look
for .kotlin_builtins files in the classpath and provide definitions of
built-ins from those files.

Also exclude script.runtime from compilation because, as other excluded
modules, it has no dependency on the stdlib and is no longer compilable from
the IDE now, because it cannot resolve built-ins from anywhere

.idea/compiler.xml

@@ -1,14 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="CompilerConfiguration">
-    <option name="DEFAULT_COMPILER" value="Javac" />
     <option name="BUILD_PROCESS_HEAP_SIZE" value="2000" />
     <excludeFromCompile>
       <directory url="file://$PROJECT_DIR$/core/reflection.jvm" includeSubdirectories="true" />
       <directory url="file://$PROJECT_DIR$/core/runtime.jvm" includeSubdirectories="true" />
       <directory url="file://$PROJECT_DIR$/core/builtins" includeSubdirectories="true" />
+      <directory url="file://$PROJECT_DIR$/core/script.runtime" includeSubdirectories="true" />
     </excludeFromCompile>
-    <resourceExtensions />
     <wildcardResourcePatterns>
       <entry name="!?*.java" />
       <entry name="!?*.form" />
@@ -19,11 +18,6 @@
       <entry name="!?*.kt" />
       <entry name="!?*.clj" />
     </wildcardResourcePatterns>
-    <annotationProcessing>
-      <profile default="true" name="Default" enabled="false">
-        <processorPath useClasspath="true" />
-      </profile>
-    </annotationProcessing>
     <bytecodeTargetLevel>
       <module name="android-studio" target="1.8" />
       <module name="idea" target="1.8" />

build.xml

@@ -961,8 +961,8 @@
                 <include name="core/runtime.jvm/src"/>
             </src>
             <class-path>
-                <!-- Need something to pass to compiler's "-classpath" here -->
+                <!-- TODO: serialize and compile built-ins in one step here instead -->
-                <pathelement location="core/builtins/src"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
     </target>
@@ -974,6 +974,7 @@
             </src>
             <class-path>
                 <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
     </target>
@@ -988,6 +989,7 @@
             <class-path>
                 <pathelement path="${output}/classes/builtins"/>
                 <pathelement path="libraries/lib/junit-4.11.jar"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
 
@@ -1012,6 +1014,7 @@
                 <pathelement path="${output}/classes/stdlib"/>
                 <pathelement path="${protobuf-lite.jar}"/>
                 <pathelement path="${javax.inject.jar}"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
     </target>
@@ -1026,6 +1029,7 @@
                 <pathelement path="${output}/classes/stdlib"/>
                 <pathelement path="${output}/classes/core"/>
                 <pathelement path="${protobuf-lite.jar}"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
     </target>
@@ -1060,7 +1064,7 @@
                 <fileset dir="${output}/classes/stdlib"/>
                 <zipfileset dir="${output}/builtins">
                     <include name="kotlin/**"/>
-                    <!-- TODO: load metadata from @KotlinClass annotation in KotlinBuiltIns on JVM and restore this exclusion -->
+                    <!-- TODO: load metadata from @Metadata annotation in KotlinBuiltIns on JVM and restore this exclusion (also below in mock-runtime) -->
                     <!-- exclude name="kotlin/reflect/**"/ -->
                 </zipfileset>
             </jar-content>
@@ -1167,6 +1171,7 @@
             </src>
             <class-path>
                 <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
         <pack-runtime-jar jar-name="kotlin-script-runtime.jar" implementation-title="${manifest.impl.title.kotlin.script.runtime}">
@@ -1225,12 +1230,17 @@
             </src>
             <class-path>
                 <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
             </class-path>
         </new-kotlinc>
 
         <pack-runtime-jar jar-dir="${output}" jar-name="kotlin-mock-runtime-for-test.jar" implementation-title="Kotlin Mock Runtime">
             <jar-content>
                 <fileset dir="${output}/classes/mock-runtime"/>
+                <zipfileset dir="${output}/builtins">
+                    <include name="kotlin/**"/>
+                    <!-- exclude name="kotlin/reflect/**"/ -->
+                </zipfileset>
             </jar-content>
         </pack-runtime-jar>
     </target>

compiler/cli/src/org/jetbrains/kotlin/cli/jvm/compiler/JvmCliVirtualFileFinder.kt

@@ -18,22 +18,36 @@ package org.jetbrains.kotlin.cli.jvm.compiler
 
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
 import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
 import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndex
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.utils.addToStdlib.check
+import java.io.InputStream
 
 class JvmCliVirtualFileFinder(
         private val index: JvmDependenciesIndex,
         private val scope: GlobalSearchScope
 ) : VirtualFileKotlinClassFinder() {
     override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? {
-        val classFileName = classId.relativeClassName.asString().replace('.', '$')
+        val classFileName = classId.relativeClassName.asString().replace('.', '$') + ".class"
         return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
-            dir.findChild("$classFileName.class")?.let {
+            dir.findChild(classFileName)?.check(VirtualFile::isValid)
-                if (it.isValid) it else null
-            }
         }?.check { it in scope }
     }
+
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? {
+        val fileName = BuiltInSerializerProtocol.getBuiltInsFileName(packageFqName)
+
+        // "<builtins-metadata>" is just a made-up name
+        // JvmDependenciesIndex requires the ClassId of the class which we're searching for, to cache the last request+result
+        val classId = ClassId(packageFqName, Name.special("<builtins-metadata>"))
+
+        return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
+            dir.findChild(fileName)?.check(VirtualFile::isValid)
+        }?.check { it in scope }?.inputStream
+    }
 }

compiler/frontend.java/src/org/jetbrains/kotlin/frontend/java/di/injection.kt

@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.frontend.java.di
 
 import com.intellij.openapi.project.Project
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.builtins.JvmBuiltInsPackageFragmentProvider
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.container.*
@@ -85,6 +86,7 @@ fun createContainerForLazyResolveWithJava(
         lookupTracker: LookupTracker,
         packagePartProvider: PackagePartProvider,
         languageVersionSettings: LanguageVersionSettings,
+        useBuiltInsProvider: Boolean,
         useLazyResolve: Boolean
 ): StorageComponentContainer = createContainer("LazyResolveWithJava", JvmPlatform) {
     configureModule(moduleContext, JvmPlatform, bindingTrace)
@@ -94,6 +96,11 @@ fun createContainerForLazyResolveWithJava(
     useInstance(moduleClassResolver)
     useInstance(declarationProviderFactory)
 
+    if (useBuiltInsProvider) {
+        useInstance((moduleContext.module.builtIns as JvmBuiltIns).settings)
+        useImpl<JvmBuiltInsPackageFragmentProvider>()
+    }
+
     targetEnvironment.configure(this)
 
     if (useLazyResolve) {
@@ -115,7 +122,8 @@ fun createContainerForTopDownAnalyzerForJvm(
         moduleClassResolver: ModuleClassResolver
 ): ComponentProvider = createContainerForLazyResolveWithJava(
         moduleContext, bindingTrace, declarationProviderFactory, moduleContentScope, moduleClassResolver,
-        CompilerEnvironment, lookupTracker, packagePartProvider, languageVersionSettings, useLazyResolve = false
+        CompilerEnvironment, lookupTracker, packagePartProvider, languageVersionSettings,
+        useBuiltInsProvider = true, useLazyResolve = false
 )
 
 

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/JvmAnalyzerFacade.kt

@@ -89,6 +89,7 @@ object JvmAnalyzerFacade : AnalyzerFacade<JvmPlatformParameters>() {
                 LookupTracker.DO_NOTHING,
                 packagePartProvider,
                 LanguageVersionSettingsImpl.DEFAULT, // TODO: see KT-12410
+                useBuiltInsProvider = false, // TODO: load built-ins from module dependencies in IDE
                 useLazyResolve = true
         )
         val resolveSession = container.get<ResolveSession>()

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/TopDownAnalyzerFacadeForJVM.kt

@@ -22,6 +22,7 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.search.DelegatingGlobalSearchScope
 import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.analyzer.AnalysisResult
+import org.jetbrains.kotlin.builtins.JvmBuiltInsPackageFragmentProvider
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
@@ -130,8 +131,11 @@ object TopDownAnalyzerFacadeForJVM {
 
             moduleClassResolver.compiledCodeResolver = dependenciesContainer.get<JavaDescriptorResolver>()
 
-            dependenciesContext.setDependencies(dependenciesContext.module, dependenciesContext.module.builtIns.builtInsModule)
+            dependenciesContext.setDependencies(dependenciesContext.module)
-            dependenciesContext.initializeModuleContents(moduleClassResolver.compiledCodeResolver.packageFragmentProvider)
+            dependenciesContext.initializeModuleContents(CompositePackageFragmentProvider(listOf(
+                    moduleClassResolver.compiledCodeResolver.packageFragmentProvider,
+                    dependenciesContainer.get<JvmBuiltInsPackageFragmentProvider>()
+            )))
             dependenciesContext.module
         }
         else null
@@ -171,7 +175,7 @@ object TopDownAnalyzerFacadeForJVM {
 
         // TODO: remove dependencyModule from friends
         module.setDependencies(ModuleDependenciesImpl(
-                listOfNotNull(module, dependencyModule, module.builtIns.builtInsModule),
+                listOfNotNull(module, dependencyModule),
                 if (dependencyModule != null) setOf(dependencyModule) else emptySet()
         ))
         module.initialize(CompositePackageFragmentProvider(

compiler/testData/cli/jvm/noStdlib.out

@@ -1,6 +1,6 @@
-compiler/testData/cli/jvm/noStdlib.kt:4:19: error: unresolved reference: primaryConstructor
+compiler/testData/cli/jvm/noStdlib.kt:1:8: error: unresolved reference: kotlin
-    String::class.primaryConstructor
+import kotlin.reflect.*
-                  ^
+       ^
 compiler/testData/cli/jvm/noStdlib.kt:5:5: error: unresolved reference: listOf
     listOf(42)
     ^

core/descriptor.loader.java/src/org/jetbrains/kotlin/builtins/JvmBuiltInsPackageFragmentProvider.kt

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.builtins
+
+import org.jetbrains.kotlin.builtins.functions.BuiltInFictitiousFunctionClassFactory
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.deserialization.*
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.utils.singletonOrEmptyList
+
+class JvmBuiltInsPackageFragmentProvider(
+        storageManager: StorageManager,
+        finder: KotlinClassFinder,
+        moduleDescriptor: ModuleDescriptor,
+        notFoundClasses: NotFoundClasses,
+        additionalClassPartsProvider: AdditionalClassPartsProvider,
+        platformDependentDeclarationFilter: PlatformDependentDeclarationFilter
+) : PackageFragmentProvider {
+    private lateinit var components: DeserializationComponents
+
+    private val fragments = storageManager.createMemoizedFunctionWithNullableValues<FqName, PackageFragmentDescriptor> { fqName ->
+        finder.findBuiltInsData(fqName)?.let { inputStream ->
+            BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor) { path -> inputStream }.apply {
+                components = this@JvmBuiltInsPackageFragmentProvider.components
+            }
+        }
+    }
+
+    init {
+        components = DeserializationComponents(
+                storageManager,
+                moduleDescriptor,
+                DeserializationConfiguration.Default,
+                DeserializedClassDataFinder(this),
+                AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol),
+                this,
+                LocalClassifierTypeSettings.Default,
+                ErrorReporter.DO_NOTHING,
+                LookupTracker.DO_NOTHING,
+                FlexibleTypeDeserializer.ThrowException,
+                listOf(
+                        BuiltInFictitiousFunctionClassFactory(storageManager, moduleDescriptor),
+                        JvmBuiltInClassDescriptorFactory(storageManager, moduleDescriptor)
+                ),
+                notFoundClasses, additionalClassPartsProvider, platformDependentDeclarationFilter
+        )
+
+    }
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> = fragments(fqName).singletonOrEmptyList()
+
+    override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection<FqName> = emptySet()
+}

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/JavaClassDataFinder.kt

@@ -22,7 +22,7 @@ import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
 
 class JavaClassDataFinder(
-        private val kotlinClassFinder: KotlinClassFinder,
+        internal val kotlinClassFinder: KotlinClassFinder,
         private val deserializedDescriptorResolver: DeserializedDescriptorResolver
 ) : ClassDataFinder {
     override fun findClassData(classId: ClassId): ClassDataWithSource? {

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/KotlinClassFinder.kt

@@ -19,9 +19,12 @@ package org.jetbrains.kotlin.load.kotlin
 import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
+import java.io.InputStream
 
 interface KotlinClassFinder {
     fun findKotlinClass(classId: ClassId): KotlinJvmBinaryClass?
 
     fun findKotlinClass(javaClass: JavaClass): KotlinJvmBinaryClass?
+
+    fun findBuiltInsData(packageFqName: FqName): InputStream?
 }

core/descriptors.runtime/src/org/jetbrains/kotlin/load/kotlin/reflect/ReflectKotlinClassFinder.kt

@@ -21,6 +21,8 @@ import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
 import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinaryClass
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import java.io.InputStream
 
 class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinClassFinder {
     private fun findKotlinClass(fqName: String): KotlinJvmBinaryClass? {
@@ -33,6 +35,9 @@ class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinCla
         // TODO: go through javaClass's class loader
         return findKotlinClass(javaClass.fqName?.asString() ?: return null)
     }
+
+    // TODO: load built-ins from classLoader
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }
 
 private fun ClassId.toRuntimeFqName(): String {

core/descriptors/src/org/jetbrains/kotlin/builtins/BuiltInSerializerProtocol.kt

@@ -30,7 +30,10 @@ object BuiltInSerializerProtocol : SerializerExtensionProtocol(
     val BUILTINS_FILE_EXTENSION = "kotlin_builtins"
 
     fun getBuiltInsFilePath(fqName: FqName): String =
-            fqName.asString().replace('.', '/') + "/" + shortName(fqName) + "." + BUILTINS_FILE_EXTENSION
+            fqName.asString().replace('.', '/') + "/" + getBuiltInsFileName(fqName)
+
+    fun getBuiltInsFileName(fqName: FqName): String =
+            shortName(fqName) + "." + BUILTINS_FILE_EXTENSION
 
     private fun shortName(fqName: FqName): String =
             if (fqName.isRoot) "default-package" else fqName.shortName().asString()

idea/idea-analysis/src/org/jetbrains/kotlin/idea/decompiler/classFile/DeserializerForClassfileDecompiler.kt

@@ -39,6 +39,7 @@ import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfigu
 import org.jetbrains.kotlin.serialization.deserialization.NotFoundClasses
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
+import java.io.InputStream
 
 fun DeserializerForClassfileDecompiler(classFile: VirtualFile): DeserializerForClassfileDecompiler {
     val kotlinClassHeaderInfo = IDEKotlinBinaryClassCache.getKotlinBinaryClassHeaderData(classFile)
@@ -114,6 +115,9 @@ class DirectoryBasedClassFinder(
         }
         return null
     }
+
+    // TODO: load built-ins from packageDirectory?
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }
 
 class DirectoryBasedDataFinder(

idea/idea-analysis/src/org/jetbrains/kotlin/idea/vfilefinder/fileFinders.kt

@@ -25,7 +25,7 @@ import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinder
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-
+import java.io.InputStream
 
 private fun checkScopeForFinder(scope: GlobalSearchScope, logger: Logger) {
     if (scope != GlobalSearchScope.EMPTY_SCOPE && scope.project == null) {
@@ -56,6 +56,8 @@ class JsIDEVirtualFileFinder(private val scope: GlobalSearchScope) : JsVirtualFi
 }
 
 class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFileKotlinClassFinder(), JvmVirtualFileFinder {
+    // TODO: load built-ins metadata from scope
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 
     init { checkScopeForFinder(scope, LOG) }
 
@@ -66,4 +68,3 @@ class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFil
         private val LOG = Logger.getInstance(JvmIDEVirtualFileFinder::class.java)
     }
 }
-