Load built-ins from module dependencies in JVM compiler

Introduce a new method KotlinClassFinder#findBuiltInsData, which is only implemented correctly in the JvmCliVirtualFileFinder because it's only used in the compiler code at the moment. Introduce JvmBuiltInsPackageFragmentProvider, the purpose of which is to look for .kotlin_builtins files in the classpath and provide definitions of built-ins from those files. Also exclude script.runtime from compilation because, as other excluded modules, it has no dependency on the stdlib and is no longer compilable from the IDE now, because it cannot resolve built-ins from anywhere
2016-10-17 18:11:38 +03:00
parent 0b59c71340
commit e0989caf46
14 changed files with 145 additions and 25 deletions
@@ -1,14 +1,13 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
  <component name="CompilerConfiguration">
-    <option name="DEFAULT_COMPILER" value="Javac" />
    <option name="BUILD_PROCESS_HEAP_SIZE" value="2000" />
    <excludeFromCompile>
      <directory url="file://$PROJECT_DIR$/core/reflection.jvm" includeSubdirectories="true" />
      <directory url="file://$PROJECT_DIR$/core/runtime.jvm" includeSubdirectories="true" />
      <directory url="file://$PROJECT_DIR$/core/builtins" includeSubdirectories="true" />
+      <directory url="file://$PROJECT_DIR$/core/script.runtime" includeSubdirectories="true" />
    </excludeFromCompile>
-    <resourceExtensions />
    <wildcardResourcePatterns>
      <entry name="!?*.java" />
      <entry name="!?*.form" />
@@ -19,11 +18,6 @@
      <entry name="!?*.kt" />
      <entry name="!?*.clj" />
    </wildcardResourcePatterns>
-    <annotationProcessing>
-      <profile default="true" name="Default" enabled="false">
-        <processorPath useClasspath="true" />
-      </profile>
-    </annotationProcessing>
    <bytecodeTargetLevel>
      <module name="android-studio" target="1.8" />
      <module name="idea" target="1.8" />
@@ -961,8 +961,8 @@
                <include name="core/runtime.jvm/src"/>
            </src>
            <class-path>
-                <!-- Need something to pass to compiler's "-classpath" here -->
-                <pathelement location="core/builtins/src"/>
+                <!-- TODO: serialize and compile built-ins in one step here instead -->
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>
    </target>
@@ -974,6 +974,7 @@
            </src>
            <class-path>
                <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>
    </target>
@@ -988,6 +989,7 @@
            <class-path>
                <pathelement path="${output}/classes/builtins"/>
                <pathelement path="libraries/lib/junit-4.11.jar"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>

@@ -1012,6 +1014,7 @@
                <pathelement path="${output}/classes/stdlib"/>
                <pathelement path="${protobuf-lite.jar}"/>
                <pathelement path="${javax.inject.jar}"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>
    </target>
@@ -1026,6 +1029,7 @@
                <pathelement path="${output}/classes/stdlib"/>
                <pathelement path="${output}/classes/core"/>
                <pathelement path="${protobuf-lite.jar}"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>
    </target>
@@ -1060,7 +1064,7 @@
                <fileset dir="${output}/classes/stdlib"/>
                <zipfileset dir="${output}/builtins">
                    <include name="kotlin/**"/>
-                    <!-- TODO: load metadata from @KotlinClass annotation in KotlinBuiltIns on JVM and restore this exclusion -->
+                    <!-- TODO: load metadata from @Metadata annotation in KotlinBuiltIns on JVM and restore this exclusion (also below in mock-runtime) -->
                    <!-- exclude name="kotlin/reflect/**"/ -->
                </zipfileset>
            </jar-content>
@@ -1167,6 +1171,7 @@
            </src>
            <class-path>
                <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>
        <pack-runtime-jar jar-name="kotlin-script-runtime.jar" implementation-title="${manifest.impl.title.kotlin.script.runtime}">
@@ -1225,12 +1230,17 @@
            </src>
            <class-path>
                <pathelement path="${output}/classes/builtins"/>
+                <pathelement path="${output}/builtins"/>
            </class-path>
        </new-kotlinc>

        <pack-runtime-jar jar-dir="${output}" jar-name="kotlin-mock-runtime-for-test.jar" implementation-title="Kotlin Mock Runtime">
            <jar-content>
                <fileset dir="${output}/classes/mock-runtime"/>
+                <zipfileset dir="${output}/builtins">
+                    <include name="kotlin/**"/>
+                    <!-- exclude name="kotlin/reflect/**"/ -->
+                </zipfileset>
            </jar-content>
        </pack-runtime-jar>
    </target>
@@ -18,22 +18,36 @@ package org.jetbrains.kotlin.cli.jvm.compiler

 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.builtins.BuiltInSerializerProtocol
 import org.jetbrains.kotlin.cli.jvm.index.JavaRoot
 import org.jetbrains.kotlin.cli.jvm.index.JvmDependenciesIndex
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.utils.addToStdlib.check
+import java.io.InputStream

 class JvmCliVirtualFileFinder(
        private val index: JvmDependenciesIndex,
        private val scope: GlobalSearchScope
 ) : VirtualFileKotlinClassFinder() {
    override fun findVirtualFileWithHeader(classId: ClassId): VirtualFile? {
-        val classFileName = classId.relativeClassName.asString().replace('.', '$')
+        val classFileName = classId.relativeClassName.asString().replace('.', '$') + ".class"
        return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
-            dir.findChild("$classFileName.class")?.let {
-                if (it.isValid) it else null
-            }
+            dir.findChild(classFileName)?.check(VirtualFile::isValid)
        }?.check { it in scope }
    }
+
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? {
+        val fileName = BuiltInSerializerProtocol.getBuiltInsFileName(packageFqName)
+
+        // "<builtins-metadata>" is just a made-up name
+        // JvmDependenciesIndex requires the ClassId of the class which we're searching for, to cache the last request+result
+        val classId = ClassId(packageFqName, Name.special("<builtins-metadata>"))
+
+        return index.findClass(classId, acceptedRootTypes = JavaRoot.OnlyBinary) { dir, rootType ->
+            dir.findChild(fileName)?.check(VirtualFile::isValid)
+        }?.check { it in scope }?.inputStream
+    }
 }
@@ -18,6 +18,7 @@ package org.jetbrains.kotlin.frontend.java.di

 import com.intellij.openapi.project.Project
 import com.intellij.psi.search.GlobalSearchScope
+import org.jetbrains.kotlin.builtins.JvmBuiltInsPackageFragmentProvider
 import org.jetbrains.kotlin.config.LanguageFeature
 import org.jetbrains.kotlin.config.LanguageVersionSettings
 import org.jetbrains.kotlin.container.*
@@ -85,6 +86,7 @@ fun createContainerForLazyResolveWithJava(
        lookupTracker: LookupTracker,
        packagePartProvider: PackagePartProvider,
        languageVersionSettings: LanguageVersionSettings,
+        useBuiltInsProvider: Boolean,
        useLazyResolve: Boolean
 ): StorageComponentContainer = createContainer("LazyResolveWithJava", JvmPlatform) {
    configureModule(moduleContext, JvmPlatform, bindingTrace)
@@ -94,6 +96,11 @@ fun createContainerForLazyResolveWithJava(
    useInstance(moduleClassResolver)
    useInstance(declarationProviderFactory)

+    if (useBuiltInsProvider) {
+        useInstance((moduleContext.module.builtIns as JvmBuiltIns).settings)
+        useImpl<JvmBuiltInsPackageFragmentProvider>()
+    }
+
    targetEnvironment.configure(this)

    if (useLazyResolve) {
@@ -115,7 +122,8 @@ fun createContainerForTopDownAnalyzerForJvm(
        moduleClassResolver: ModuleClassResolver
 ): ComponentProvider = createContainerForLazyResolveWithJava(
        moduleContext, bindingTrace, declarationProviderFactory, moduleContentScope, moduleClassResolver,
-        CompilerEnvironment, lookupTracker, packagePartProvider, languageVersionSettings, useLazyResolve = false
+        CompilerEnvironment, lookupTracker, packagePartProvider, languageVersionSettings,
+        useBuiltInsProvider = true, useLazyResolve = false
 )


@@ -89,6 +89,7 @@ object JvmAnalyzerFacade : AnalyzerFacade<JvmPlatformParameters>() {
                LookupTracker.DO_NOTHING,
                packagePartProvider,
                LanguageVersionSettingsImpl.DEFAULT, // TODO: see KT-12410
+                useBuiltInsProvider = false, // TODO: load built-ins from module dependencies in IDE
                useLazyResolve = true
        )
        val resolveSession = container.get<ResolveSession>()
@@ -22,6 +22,7 @@ import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.psi.search.DelegatingGlobalSearchScope
 import com.intellij.psi.search.GlobalSearchScope
 import org.jetbrains.kotlin.analyzer.AnalysisResult
+import org.jetbrains.kotlin.builtins.JvmBuiltInsPackageFragmentProvider
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.JVMConfigurationKeys
@@ -130,8 +131,11 @@ object TopDownAnalyzerFacadeForJVM {

            moduleClassResolver.compiledCodeResolver = dependenciesContainer.get<JavaDescriptorResolver>()

-            dependenciesContext.setDependencies(dependenciesContext.module, dependenciesContext.module.builtIns.builtInsModule)
-            dependenciesContext.initializeModuleContents(moduleClassResolver.compiledCodeResolver.packageFragmentProvider)
+            dependenciesContext.setDependencies(dependenciesContext.module)
+            dependenciesContext.initializeModuleContents(CompositePackageFragmentProvider(listOf(
+                    moduleClassResolver.compiledCodeResolver.packageFragmentProvider,
+                    dependenciesContainer.get<JvmBuiltInsPackageFragmentProvider>()
+            )))
            dependenciesContext.module
        }
        else null
@@ -171,7 +175,7 @@ object TopDownAnalyzerFacadeForJVM {

        // TODO: remove dependencyModule from friends
        module.setDependencies(ModuleDependenciesImpl(
-                listOfNotNull(module, dependencyModule, module.builtIns.builtInsModule),
+                listOfNotNull(module, dependencyModule),
                if (dependencyModule != null) setOf(dependencyModule) else emptySet()
        ))
        module.initialize(CompositePackageFragmentProvider(
@@ -1,6 +1,6 @@
-compiler/testData/cli/jvm/noStdlib.kt:4:19: error: unresolved reference: primaryConstructor
-    String::class.primaryConstructor
-                  ^
+compiler/testData/cli/jvm/noStdlib.kt:1:8: error: unresolved reference: kotlin
+import kotlin.reflect.*
+       ^
 compiler/testData/cli/jvm/noStdlib.kt:5:5: error: unresolved reference: listOf
    listOf(42)
    ^
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.builtins
+
+import org.jetbrains.kotlin.builtins.functions.BuiltInFictitiousFunctionClassFactory
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentProvider
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.serialization.deserialization.*
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.utils.singletonOrEmptyList
+
+class JvmBuiltInsPackageFragmentProvider(
+        storageManager: StorageManager,
+        finder: KotlinClassFinder,
+        moduleDescriptor: ModuleDescriptor,
+        notFoundClasses: NotFoundClasses,
+        additionalClassPartsProvider: AdditionalClassPartsProvider,
+        platformDependentDeclarationFilter: PlatformDependentDeclarationFilter
+) : PackageFragmentProvider {
+    private lateinit var components: DeserializationComponents
+
+    private val fragments = storageManager.createMemoizedFunctionWithNullableValues<FqName, PackageFragmentDescriptor> { fqName ->
+        finder.findBuiltInsData(fqName)?.let { inputStream ->
+            BuiltInsPackageFragment(fqName, storageManager, moduleDescriptor) { path -> inputStream }.apply {
+                components = this@JvmBuiltInsPackageFragmentProvider.components
+            }
+        }
+    }
+
+    init {
+        components = DeserializationComponents(
+                storageManager,
+                moduleDescriptor,
+                DeserializationConfiguration.Default,
+                DeserializedClassDataFinder(this),
+                AnnotationAndConstantLoaderImpl(moduleDescriptor, notFoundClasses, BuiltInSerializerProtocol),
+                this,
+                LocalClassifierTypeSettings.Default,
+                ErrorReporter.DO_NOTHING,
+                LookupTracker.DO_NOTHING,
+                FlexibleTypeDeserializer.ThrowException,
+                listOf(
+                        BuiltInFictitiousFunctionClassFactory(storageManager, moduleDescriptor),
+                        JvmBuiltInClassDescriptorFactory(storageManager, moduleDescriptor)
+                ),
+                notFoundClasses, additionalClassPartsProvider, platformDependentDeclarationFilter
+        )
+
+    }
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> = fragments(fqName).singletonOrEmptyList()
+
+    override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection<FqName> = emptySet()
+}
@@ -22,7 +22,7 @@ import org.jetbrains.kotlin.serialization.deserialization.ClassDataFinder
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil

 class JavaClassDataFinder(
-        private val kotlinClassFinder: KotlinClassFinder,
+        internal val kotlinClassFinder: KotlinClassFinder,
        private val deserializedDescriptorResolver: DeserializedDescriptorResolver
 ) : ClassDataFinder {
    override fun findClassData(classId: ClassId): ClassDataWithSource? {
@@ -19,9 +19,12 @@ package org.jetbrains.kotlin.load.kotlin
 import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
+import java.io.InputStream

 interface KotlinClassFinder {
    fun findKotlinClass(classId: ClassId): KotlinJvmBinaryClass?

    fun findKotlinClass(javaClass: JavaClass): KotlinJvmBinaryClass?
+
+    fun findBuiltInsData(packageFqName: FqName): InputStream?
 }
@@ -21,6 +21,8 @@ import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.kotlin.KotlinClassFinder
 import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinaryClass
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import java.io.InputStream

 class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinClassFinder {
    private fun findKotlinClass(fqName: String): KotlinJvmBinaryClass? {
@@ -33,6 +35,9 @@ class ReflectKotlinClassFinder(private val classLoader: ClassLoader) : KotlinCla
        // TODO: go through javaClass's class loader
        return findKotlinClass(javaClass.fqName?.asString() ?: return null)
    }
+
+    // TODO: load built-ins from classLoader
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }

 private fun ClassId.toRuntimeFqName(): String {
@@ -30,7 +30,10 @@ object BuiltInSerializerProtocol : SerializerExtensionProtocol(
    val BUILTINS_FILE_EXTENSION = "kotlin_builtins"

    fun getBuiltInsFilePath(fqName: FqName): String =
-            fqName.asString().replace('.', '/') + "/" + shortName(fqName) + "." + BUILTINS_FILE_EXTENSION
+            fqName.asString().replace('.', '/') + "/" + getBuiltInsFileName(fqName)
+
+    fun getBuiltInsFileName(fqName: FqName): String =
+            shortName(fqName) + "." + BUILTINS_FILE_EXTENSION

    private fun shortName(fqName: FqName): String =
            if (fqName.isRoot) "default-package" else fqName.shortName().asString()
@@ -39,6 +39,7 @@ import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfigu
 import org.jetbrains.kotlin.serialization.deserialization.NotFoundClasses
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedPackageMemberScope
 import org.jetbrains.kotlin.serialization.jvm.JvmProtoBufUtil
+import java.io.InputStream

 fun DeserializerForClassfileDecompiler(classFile: VirtualFile): DeserializerForClassfileDecompiler {
    val kotlinClassHeaderInfo = IDEKotlinBinaryClassCache.getKotlinBinaryClassHeaderData(classFile)
@@ -114,6 +115,9 @@ class DirectoryBasedClassFinder(
        }
        return null
    }
+
+    // TODO: load built-ins from packageDirectory?
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null
 }

 class DirectoryBasedDataFinder(
@@ -25,7 +25,7 @@ import org.jetbrains.kotlin.load.kotlin.JvmVirtualFileFinder
 import org.jetbrains.kotlin.load.kotlin.VirtualFileKotlinClassFinder
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
-
+import java.io.InputStream

 private fun checkScopeForFinder(scope: GlobalSearchScope, logger: Logger) {
    if (scope != GlobalSearchScope.EMPTY_SCOPE && scope.project == null) {
@@ -56,6 +56,8 @@ class JsIDEVirtualFileFinder(private val scope: GlobalSearchScope) : JsVirtualFi
 }

 class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFileKotlinClassFinder(), JvmVirtualFileFinder {
+    // TODO: load built-ins metadata from scope
+    override fun findBuiltInsData(packageFqName: FqName): InputStream? = null

    init { checkScopeForFinder(scope, LOG) }

@@ -66,4 +68,3 @@ class JvmIDEVirtualFileFinder(private val scope: GlobalSearchScope) : VirtualFil
        private val LOG = Logger.getInstance(JvmIDEVirtualFileFinder::class.java)
    }
 }
-