Do not use .kotlin_module files in reflection

Previously, we used a pretty roundabout way to load a MemberScope from a
single file facade represented by KPackageImpl, which involved going
through ModuleDescriptor, PackageFragmentProvider, PackagePartProvider
etc. The only advantage of this approach was that it sort of works
similarly as in the compiler, however mutable state in
RuntimePackagePartProvider and the fact that .kotlin_module files were
required for this to work diminished this advantage.

In this change, we load MemberScope from a KPackageImpl pretty much
directly, by using the existing method
`DeserializedDescriptorResolver.createKotlinPackagePartScope` and
caching the result in the new component PackagePartScopeCache.

 #KT-30344 Fixed

core/descriptors.runtime/src/kotlin/reflect/jvm/internal/components/PackagePartScopeCache.kt

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the license/LICENSE.txt file.
+ */
+
+package kotlin.reflect.jvm.internal.components
+
+import org.jetbrains.kotlin.descriptors.impl.EmptyPackageFragmentDescriptor
+import org.jetbrains.kotlin.load.kotlin.DeserializedDescriptorResolver
+import org.jetbrains.kotlin.load.kotlin.findKotlinClass
+import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.resolve.jvm.JvmClassName
+import org.jetbrains.kotlin.resolve.scopes.ChainedMemberScope
+import org.jetbrains.kotlin.resolve.scopes.MemberScope
+import java.util.concurrent.ConcurrentHashMap
+
+class PackagePartScopeCache(private val resolver: DeserializedDescriptorResolver, private val kotlinClassFinder: ReflectKotlinClassFinder) {
+    private val cache = ConcurrentHashMap<ClassId, MemberScope>()
+
+    fun getPackagePartScope(fileClass: ReflectKotlinClass): MemberScope = cache.getOrPut(fileClass.classId) {
+        val fqName = fileClass.classId.packageFqName
+
+        val parts =
+            if (fileClass.classHeader.kind == KotlinClassHeader.Kind.MULTIFILE_CLASS)
+                fileClass.classHeader.multifilePartNames.mapNotNull { partName ->
+                    val classId = ClassId.topLevel(JvmClassName.byInternalName(partName).fqNameForTopLevelClassMaybeWithDollars)
+                    kotlinClassFinder.findKotlinClass(classId)
+                }
+            else listOf(fileClass)
+
+        val packageFragment = EmptyPackageFragmentDescriptor(resolver.components.moduleDescriptor, fqName)
+
+        val scopes = parts.mapNotNull { part ->
+            resolver.createKotlinPackagePartScope(packageFragment, part)
+        }.toList()
+
+        ChainedMemberScope.create("package $fqName ($fileClass)", scopes)
+    }
+}

core/descriptors.runtime/src/kotlin/reflect/jvm/internal/components/RuntimeModuleData.kt

@@ -36,10 +36,7 @@ import org.jetbrains.kotlin.load.java.lazy.JavaResolverSettings
 import org.jetbrains.kotlin.load.java.lazy.LazyJavaPackageFragmentProvider
 import org.jetbrains.kotlin.load.java.lazy.SingleModuleClassResolver
 import org.jetbrains.kotlin.load.java.typeEnhancement.SignatureEnhancement
-import org.jetbrains.kotlin.load.kotlin.BinaryClassAnnotationAndConstantLoaderImpl
-import org.jetbrains.kotlin.load.kotlin.DeserializationComponentsForJava
-import org.jetbrains.kotlin.load.kotlin.DeserializedDescriptorResolver
-import org.jetbrains.kotlin.load.kotlin.JavaClassDataFinder
+import org.jetbrains.kotlin.load.kotlin.*
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.jvm.JavaDescriptorResolver
 import org.jetbrains.kotlin.serialization.deserialization.ContractDeserializer
@@ -50,7 +47,7 @@ import org.jetbrains.kotlin.utils.Jsr305State
 
 class RuntimeModuleData private constructor(
     val deserialization: DeserializationComponents,
-    val packagePartProvider: RuntimePackagePartProvider
+    val packagePartScopeCache: PackagePartScopeCache
 ) {
     val module: ModuleDescriptor get() = deserialization.moduleDescriptor
 
@@ -64,25 +61,23 @@ class RuntimeModuleData private constructor(
             val reflectKotlinClassFinder = ReflectKotlinClassFinder(classLoader)
             val deserializedDescriptorResolver = DeserializedDescriptorResolver()
             val singleModuleClassResolver = SingleModuleClassResolver()
-            val runtimePackagePartProvider = RuntimePackagePartProvider(classLoader)
-            val javaResolverCache = JavaResolverCache.EMPTY
             val notFoundClasses = NotFoundClasses(storageManager, module)
             val annotationTypeQualifierResolver = AnnotationTypeQualifierResolver(storageManager, Jsr305State.DISABLED)
-            val globalJavaResolverContext = JavaResolverComponents(
+            val javaResolverComponents = JavaResolverComponents(
                 storageManager, ReflectJavaClassFinder(classLoader), reflectKotlinClassFinder, deserializedDescriptorResolver,
-                SignaturePropagator.DO_NOTHING, RuntimeErrorReporter, javaResolverCache,
+                SignaturePropagator.DO_NOTHING, RuntimeErrorReporter, JavaResolverCache.EMPTY,
                 JavaPropertyInitializerEvaluator.DoNothing, SamConversionResolver.Empty, RuntimeSourceElementFactory,
-                singleModuleClassResolver, runtimePackagePartProvider, SupertypeLoopChecker.EMPTY, LookupTracker.DO_NOTHING, module,
+                singleModuleClassResolver, PackagePartProvider.Empty, SupertypeLoopChecker.EMPTY, LookupTracker.DO_NOTHING, module,
                 ReflectionTypes(module, notFoundClasses), annotationTypeQualifierResolver,
                 SignatureEnhancement(annotationTypeQualifierResolver, Jsr305State.DISABLED),
                 JavaClassesTracker.Default, JavaResolverSettings.Default
             )
 
-            val lazyJavaPackageFragmentProvider = LazyJavaPackageFragmentProvider(globalJavaResolverContext)
+            val lazyJavaPackageFragmentProvider = LazyJavaPackageFragmentProvider(javaResolverComponents)
 
             builtIns.initialize(module, isAdditionalBuiltInsFeatureSupported = true)
 
-            val javaDescriptorResolver = JavaDescriptorResolver(lazyJavaPackageFragmentProvider, javaResolverCache)
+            val javaDescriptorResolver = JavaDescriptorResolver(lazyJavaPackageFragmentProvider, JavaResolverCache.EMPTY)
             val javaClassDataFinder = JavaClassDataFinder(reflectKotlinClassFinder, deserializedDescriptorResolver)
             val binaryClassAnnotationAndConstantLoader = BinaryClassAnnotationAndConstantLoaderImpl(
                 module, notFoundClasses, storageManager, reflectKotlinClassFinder
@@ -103,7 +98,10 @@ class RuntimeModuleData private constructor(
             module.setDependencies(module)
             module.initialize(CompositePackageFragmentProvider(listOf(javaDescriptorResolver.packageFragmentProvider, builtinsProvider)))
 
-            return RuntimeModuleData(deserializationComponentsForJava.components, runtimePackagePartProvider)
+            return RuntimeModuleData(
+                deserializationComponentsForJava.components,
+                PackagePartScopeCache(deserializedDescriptorResolver, reflectKotlinClassFinder)
+            )
         }
     }
 }

core/descriptors.runtime/src/kotlin/reflect/jvm/internal/components/RuntimePackagePartProvider.kt

@@ -1,82 +0,0 @@
-/*
- * Copyright 2010-2015 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package kotlin.reflect.jvm.internal.components
-
-import org.jetbrains.kotlin.load.kotlin.PackagePartProvider
-import org.jetbrains.kotlin.load.kotlin.loadModuleMapping
-import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmMetadataVersion
-import org.jetbrains.kotlin.metadata.jvm.deserialization.ModuleMapping
-import org.jetbrains.kotlin.name.ClassId
-import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
-import java.io.IOException
-import java.util.*
-
-class RuntimePackagePartProvider(private val classLoader: ClassLoader) : PackagePartProvider {
-    // Names of modules which were registered with registerModule
-    private val visitedModules = hashSetOf<String>()
-
-    // Package FQ name -> list of JVM internal names of package parts in that package across all registered modules
-    private val packageParts = hashMapOf<String, LinkedHashSet<String>>()
-
-    @Synchronized
-    fun registerModule(moduleName: String) {
-        if (!visitedModules.add(moduleName)) return
-
-        val resourcePath = "META-INF/$moduleName.${ModuleMapping.MAPPING_FILE_EXT}"
-        val resources = try {
-            classLoader.getResources(resourcePath)
-        } catch (e: IOException) {
-            EmptyEnumeration
-        }
-
-        for (resource in resources) {
-            try {
-                resource.openStream()?.use { stream ->
-                    val mapping = ModuleMapping.loadModuleMapping(
-                        stream.readBytes(), resourcePath, DeserializationConfiguration.Default
-                    ) { version ->
-                        throw UnsupportedOperationException(
-                            "Module was compiled with an incompatible version of Kotlin. The binary version of its metadata is $version, " +
-                                    "expected version is ${JvmMetadataVersion.INSTANCE}. Please update Kotlin to the latest version"
-                        )
-                    }
-                    for ((packageFqName, parts) in mapping.packageFqName2Parts) {
-                        packageParts.getOrPut(packageFqName) { linkedSetOf() }.addAll(parts.parts)
-                    }
-                }
-            } catch (e: UnsupportedOperationException) {
-                throw e
-            } catch (e: Exception) {
-                // TODO: do not swallow this exception?
-            }
-        }
-    }
-
-    @Synchronized
-    override fun findPackageParts(packageFqName: String): List<String> =
-        packageParts[packageFqName]?.toList().orEmpty()
-
-    override fun getAnnotationsOnBinaryModule(moduleName: String): List<ClassId> {
-        // TODO: load annotations from resource files
-        return emptyList()
-    }
-
-    private object EmptyEnumeration : Enumeration<Nothing> {
-        override fun hasMoreElements(): Boolean = false
-        override fun nextElement(): Nothing = throw NoSuchElementException()
-    }
-}

core/descriptors.runtime/tests/org/jetbrains/kotlin/jvm/runtime/AbstractJvmRuntimeDescriptorLoaderTest.kt

@@ -78,7 +78,7 @@ abstract class AbstractJvmRuntimeDescriptorLoaderTest : TestCaseWithTmpdir() {
 
         val classLoader = URLClassLoader(arrayOf(tmpdir.toURI().toURL()), ForTestCompileRuntime.runtimeAndReflectJarClassLoader())
 
-        val actual = createReflectedPackageView(classLoader, KotlinTestUtils.TEST_MODULE_NAME)
+        val actual = createReflectedPackageView(classLoader)
 
         val comparatorConfiguration = Configuration(
             /* checkPrimaryConstructors = */ fileName.endsWith(".kt"),
@@ -135,9 +135,8 @@ abstract class AbstractJvmRuntimeDescriptorLoaderTest : TestCaseWithTmpdir() {
         }
     }
 
-    private fun createReflectedPackageView(classLoader: URLClassLoader, moduleName: String): SyntheticPackageViewForTest {
+    private fun createReflectedPackageView(classLoader: URLClassLoader): SyntheticPackageViewForTest {
         val moduleData = RuntimeModuleData.create(classLoader)
-        moduleData.packagePartProvider.registerModule(moduleName)
         val module = moduleData.module
 
         val generatedPackageDir = File(tmpdir, LoadDescriptorUtil.TEST_PACKAGE_FQNAME.pathSegments().single().asString())
@@ -153,10 +152,7 @@ abstract class AbstractJvmRuntimeDescriptorLoaderTest : TestCaseWithTmpdir() {
             val header = binaryClass?.classHeader
 
             if (header?.kind == KotlinClassHeader.Kind.FILE_FACADE || header?.kind == KotlinClassHeader.Kind.MULTIFILE_CLASS) {
-                val packageView = module.getPackage(LoadDescriptorUtil.TEST_PACKAGE_FQNAME)
-                if (!packageScopes.contains(packageView.memberScope)) {
-                    packageScopes.add(packageView.memberScope)
-                }
+                packageScopes.add(moduleData.packagePartScopeCache.getPackagePartScope(binaryClass))
             } else if (header == null || header.kind == KotlinClassHeader.Kind.CLASS) {
                 // Either a normal Kotlin class or a Java class
                 val classId = klass.classId

core/reflection.jvm/src/kotlin/reflect/jvm/internal/KPackageImpl.kt

@@ -16,10 +16,10 @@
 
 package kotlin.reflect.jvm.internal
 
-import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.ConstructorDescriptor
+import org.jetbrains.kotlin.descriptors.FunctionDescriptor
+import org.jetbrains.kotlin.descriptors.PropertyDescriptor
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
-import org.jetbrains.kotlin.load.java.lazy.descriptors.LazyJavaPackageFragment
-import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinaryPackageSourceElement
 import org.jetbrains.kotlin.metadata.ProtoBuf
 import org.jetbrains.kotlin.metadata.deserialization.TypeTable
 import org.jetbrains.kotlin.metadata.deserialization.getExtensionOrNull
@@ -30,7 +30,6 @@ import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.serialization.deserialization.MemberDeserializer
-import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedCallableMemberDescriptor
 import kotlin.reflect.KCallable
 import kotlin.reflect.jvm.internal.KDeclarationContainerImpl.MemberBelonginess.DECLARED
 import kotlin.reflect.jvm.internal.components.ReflectKotlinClass
@@ -42,26 +41,24 @@ internal class KPackageImpl(
 ) : KDeclarationContainerImpl() {
     private inner class Data : KDeclarationContainerImpl.Data() {
         private val kotlinClass: ReflectKotlinClass? by ReflectProperties.lazySoft {
-            // TODO: do not read ReflectKotlinClass multiple times
             ReflectKotlinClass.create(jClass)
         }
 
-        val descriptor: PackageViewDescriptor by ReflectProperties.lazySoft {
-            with(moduleData) {
-                kotlinClass?.packageModuleName?.let(packagePartProvider::registerModule)
-                module.getPackage(jClass.classId.packageFqName)
-            }
+        val scope: MemberScope by ReflectProperties.lazySoft {
+            val klass = kotlinClass
+
+            if (klass != null)
+                moduleData.packagePartScopeCache.getPackagePartScope(klass)
+            else MemberScope.Empty
         }
 
-        val methodOwner: Class<*> by ReflectProperties.lazy {
+        val multifileFacade: Class<*>? by ReflectProperties.lazy {
             val facadeName = kotlinClass?.classHeader?.multifileClassName
             // We need to check isNotEmpty because this is the value read from the annotation which cannot be null.
             // The default value for 'xs' is empty string, as declared in kotlin.Metadata
-            if (facadeName != null && facadeName.isNotEmpty()) {
+            if (facadeName != null && facadeName.isNotEmpty())
                 jClass.classLoader.loadClass(facadeName.replace('/', '.'))
-            } else {
-                jClass
-            }
+            else null
         }
 
         val metadata: Triple<JvmNameResolver, ProtoBuf.Package, JvmMetadataVersion>? by ReflectProperties.lazy {
@@ -76,20 +73,15 @@ internal class KPackageImpl(
         }
 
         val members: Collection<KCallableImpl<*>> by ReflectProperties.lazySoft {
-            getMembers(scope, DECLARED).filter { member ->
-                val callableDescriptor = member.descriptor as DeserializedCallableMemberDescriptor
-                val packageFragment = callableDescriptor.containingDeclaration as PackageFragmentDescriptor
-                val source = (packageFragment as? LazyJavaPackageFragment)?.source as? KotlinJvmBinaryPackageSourceElement
-                (source?.getContainingBinaryClass(callableDescriptor) as? ReflectKotlinClass)?.klass == jClass
-            }
+            getMembers(scope, DECLARED)
         }
     }
 
     private val data = ReflectProperties.lazy { Data() }
 
-    override val methodOwner: Class<*> get() = data().methodOwner
+    override val methodOwner: Class<*> get() = data().multifileFacade ?: jClass
 
-    private val scope: MemberScope get() = data().descriptor.memberScope
+    private val scope: MemberScope get() = data().scope
 
     override val members: Collection<KCallable<*>> get() = data().members
 
@@ -119,8 +111,6 @@ internal class KPackageImpl(
     override fun hashCode(): Int =
         jClass.hashCode()
 
-    override fun toString(): String {
-        val fqName = jClass.classId.packageFqName
-        return "package " + (if (fqName.isRoot) "<default>" else fqName.asString())
-    }
+    override fun toString(): String =
+        "file class ${jClass.classId.asSingleFqName()}"
 }

core/reflection.jvm/src/kotlin/reflect/jvm/internal/util.kt

@@ -20,13 +20,12 @@ import org.jetbrains.kotlin.builtins.jvm.JavaToKotlinClassMap
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.annotations.Annotated
 import org.jetbrains.kotlin.descriptors.annotations.AnnotationDescriptor
-import org.jetbrains.kotlin.load.java.JvmAbi
 import org.jetbrains.kotlin.load.kotlin.KotlinJvmBinarySourceElement
-import org.jetbrains.kotlin.load.kotlin.header.KotlinClassHeader
 import org.jetbrains.kotlin.metadata.ProtoBuf
-import org.jetbrains.kotlin.metadata.deserialization.*
-import org.jetbrains.kotlin.metadata.jvm.JvmProtoBuf
-import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil
+import org.jetbrains.kotlin.metadata.deserialization.BinaryVersion
+import org.jetbrains.kotlin.metadata.deserialization.NameResolver
+import org.jetbrains.kotlin.metadata.deserialization.TypeTable
+import org.jetbrains.kotlin.metadata.deserialization.VersionRequirementTable
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.protobuf.MessageLite
@@ -169,27 +168,6 @@ internal fun Any?.asKPropertyImpl(): KPropertyImpl<*>? =
 internal fun Any?.asKCallableImpl(): KCallableImpl<*>? =
     this as? KCallableImpl<*> ?: asKFunctionImpl() ?: asKPropertyImpl()
 
-internal val ReflectKotlinClass.packageModuleName: String?
-    get() {
-        val header = classHeader
-        if (!header.metadataVersion.isCompatible()) return null
-
-        return when (header.kind) {
-            KotlinClassHeader.Kind.FILE_FACADE, KotlinClassHeader.Kind.MULTIFILE_CLASS_PART -> {
-                // TODO: avoid reading and parsing metadata twice (here and later in KPackageImpl#descriptor)
-                val (nameResolver, proto) = JvmProtoBufUtil.readPackageDataFrom(header.data!!, header.strings!!)
-                // If no packageModuleName extension is written, the name is assumed to be JvmAbi.DEFAULT_MODULE_NAME
-                // (see JvmSerializerExtension.serializePackage)
-                proto.getExtensionOrNull(JvmProtoBuf.packageModuleName)?.let(nameResolver::getString) ?: JvmAbi.DEFAULT_MODULE_NAME
-            }
-            KotlinClassHeader.Kind.MULTIFILE_CLASS -> {
-                val partName = header.multifilePartNames.firstOrNull() ?: return null
-                ReflectKotlinClass.create(klass.classLoader.loadClass(partName.replace('/', '.')))?.packageModuleName
-            }
-            else -> null
-        }
-    }
-
 internal val CallableDescriptor.instanceReceiverParameter: ReceiverParameterDescriptor?
     get() =
         if (dispatchReceiverParameter != null) (containingDeclaration as ClassDescriptor).thisAsReceiverParameter