Load additional JDK functions into built-ins member scope

#KT-5990 Fixed
 #KT-7127 Fixed
 #KT-10370 Fixed

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/java/lazy/context.kt

@@ -48,7 +48,15 @@ class JavaResolverComponents(
         val packageMapper: PackagePartProvider,
         val supertypeLoopChecker: SupertypeLoopChecker,
         val lookupTracker: LookupTracker
-)
+) {
+    fun replace(
+            javaResolverCache: JavaResolverCache = this.javaResolverCache
+    ) = JavaResolverComponents(
+            storageManager, finder, kotlinClassFinder, deserializedDescriptorResolver,
+            externalAnnotationResolver, signaturePropagator, errorReporter, javaResolverCache,
+            javaPropertyInitializerEvaluator, samConversionResolver, sourceElementFactory,
+            moduleClassResolver, packageMapper, supertypeLoopChecker, lookupTracker)
+}
 
 open class LazyJavaResolverContext(
         val components: JavaResolverComponents,
@@ -68,6 +76,9 @@ fun LazyJavaResolverContext.child(
         typeParameterResolver: TypeParameterResolver
 ) = LazyJavaResolverContext(components, packageFragmentProvider, javaClassResolver, module, reflectionTypes, typeParameterResolver)
 
+fun LazyJavaResolverContext.replaceComponents(
+        components: JavaResolverComponents
+) = LazyJavaResolverContext(components, packageFragmentProvider, javaClassResolver, module, reflectionTypes, typeParameterResolver)
 
 fun LazyJavaResolverContext.child(
         containingDeclaration: DeclarationDescriptor,

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/java/lazy/descriptors/LazyJavaClassDescriptor.kt

@@ -23,17 +23,17 @@ import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorBase
 import org.jetbrains.kotlin.load.java.FakePureImplementationsProvider
 import org.jetbrains.kotlin.load.java.JavaVisibilities
 import org.jetbrains.kotlin.load.java.JvmAnnotationNames
+import org.jetbrains.kotlin.load.java.components.JavaResolverCache
 import org.jetbrains.kotlin.load.java.components.TypeUsage
 import org.jetbrains.kotlin.load.java.descriptors.JavaClassDescriptor
-import org.jetbrains.kotlin.load.java.lazy.LazyJavaResolverContext
-import org.jetbrains.kotlin.load.java.lazy.child
-import org.jetbrains.kotlin.load.java.lazy.resolveAnnotations
+import org.jetbrains.kotlin.load.java.lazy.*
 import org.jetbrains.kotlin.load.java.lazy.types.toAttributes
 import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.load.java.structure.JavaClassifierType
 import org.jetbrains.kotlin.load.java.structure.JavaType
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.isValidJavaFqName
+import org.jetbrains.kotlin.platform.createMappedTypeParametersSubstitution
 import org.jetbrains.kotlin.resolve.constants.StringValue
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameUnsafe
@@ -48,7 +48,8 @@ import java.util.*
 class LazyJavaClassDescriptor(
         private val outerContext: LazyJavaResolverContext,
         containingDeclaration: DeclarationDescriptor,
-        private val jClass: JavaClass
+        private val jClass: JavaClass,
+        private val additionalSupertypeClassDescriptor: ClassDescriptor? = null
 ) : ClassDescriptorBase(outerContext.storageManager, containingDeclaration, jClass.name,
                         outerContext.components.sourceElementFactory.source(jClass)), JavaClassDescriptor {
 
@@ -157,6 +158,14 @@ class LazyJavaClassDescriptor(
                 }
             }
 
+            // Add fake supertype kotlin.collection.Collection<E> to java.util.Collection<E> class if needed
+            // Only needed when calculating built-ins member scope
+            result.addIfNotNull(
+                    additionalSupertypeClassDescriptor?.let {
+                        createMappedTypeParametersSubstitution(it, this@LazyJavaClassDescriptor)
+                                .buildSubstitutor().substitute(it.defaultType, Variance.INVARIANT)
+                    })
+
             result.addIfNotNull(purelyImplementedSupertype)
 
             if (incomplete.isNotEmpty()) {
@@ -213,4 +222,11 @@ class LazyJavaClassDescriptor(
 
         override fun toString(): String = getName().asString()
     }
+
+    // Only needed when calculating built-ins member scope
+    internal fun copy(
+            javaResolverCache: JavaResolverCache, additionalSupertypeClassDescriptor: ClassDescriptor?
+    ) = LazyJavaClassDescriptor(
+            c.replaceComponents(c.components.replace(javaResolverCache = javaResolverCache)),
+            containingDeclaration, jClass, additionalSupertypeClassDescriptor)
 }

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/java/lazy/descriptors/LazyJavaClassMemberScope.kt

@@ -82,7 +82,7 @@ class LazyJavaClassMemberScope(
             result.add(descriptor)
             result.addIfNotNull(c.components.samConversionResolver.resolveSamAdapter(descriptor))
         }
-        
+
         enhanceSignatures(
                 result.ifEmpty { emptyOrSingletonList(createDefaultConstructor()) }
         ).toReadOnlyList()
@@ -346,10 +346,10 @@ class LazyJavaClassMemberScope(
     }
 
     private fun getFunctionsFromSupertypes(name: Name): Set<SimpleFunctionDescriptor> {
-          return ownerDescriptor.typeConstructor.supertypes.flatMapTo(LinkedHashSet()) {
-              it.memberScope.getContributedFunctions(name, NoLookupLocation.WHEN_GET_SUPER_MEMBERS)
-          }
-      }
+        return ownerDescriptor.typeConstructor.supertypes.flatMapTo(LinkedHashSet()) {
+            it.memberScope.getContributedFunctions(name, NoLookupLocation.WHEN_GET_SUPER_MEMBERS)
+        }
+    }
 
     override fun computeNonDeclaredProperties(name: Name, result: MutableCollection<PropertyDescriptor>) {
         if (jClass.isAnnotationType) {

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/BuiltInClassesAreSerializableOnJvm.kt

@@ -17,28 +17,42 @@
 package org.jetbrains.kotlin.load.kotlin
 
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
-import org.jetbrains.kotlin.descriptors.ClassKind
-import org.jetbrains.kotlin.descriptors.Modality
-import org.jetbrains.kotlin.descriptors.ModuleDescriptor
-import org.jetbrains.kotlin.descriptors.SourceElement
+import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
+import org.jetbrains.kotlin.incremental.components.NoLookupLocation
+import org.jetbrains.kotlin.load.java.components.JavaResolverCache
+import org.jetbrains.kotlin.load.java.lazy.descriptors.LazyJavaClassDescriptor
+import org.jetbrains.kotlin.load.java.lazy.descriptors.LazyJavaClassMemberScope
+import org.jetbrains.kotlin.load.java.lazy.replaceComponents
+import org.jetbrains.kotlin.load.java.sources.JavaSourceElement
+import org.jetbrains.kotlin.load.java.structure.JavaClass
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.platform.JavaToKotlinClassMap
-import org.jetbrains.kotlin.platform.JvmBuiltIns
+import org.jetbrains.kotlin.platform.createMappedTypeParametersSubstitution
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
+import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameUnsafe
+import org.jetbrains.kotlin.resolve.jvm.JvmPrimitiveType
+import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
 import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.serialization.deserialization.AdditionalClassPartsProvider
 import org.jetbrains.kotlin.serialization.deserialization.descriptors.DeserializedClassDescriptor
 import org.jetbrains.kotlin.types.DelegatingType
 import org.jetbrains.kotlin.types.KotlinType
+import org.jetbrains.kotlin.types.Variance
+import org.jetbrains.kotlin.utils.SmartSet
+import org.jetbrains.kotlin.utils.addToStdlib.check
 import java.io.Serializable
+import java.util.*
 
-class BuiltInClassesAreSerializableOnJvm(
-        private val moduleDescriptor: ModuleDescriptor
+open class BuiltInClassesAreSerializableOnJvm(
+        private val moduleDescriptor: ModuleDescriptor,
+        deferredOwnerModuleDescriptor: () -> ModuleDescriptor
 ) : AdditionalClassPartsProvider {
 
+    private val ownerModuleDescriptor: ModuleDescriptor by lazy(deferredOwnerModuleDescriptor)
+
     private val mockSerializableType = createMockJavaIoSerializableType()
 
     private fun createMockJavaIoSerializableType(): KotlinType {
@@ -49,7 +63,7 @@ class BuiltInClassesAreSerializableOnJvm(
         //NOTE: can't reference anyType right away, because this is sometimes called when JvmBuiltIns are initializing
         val superTypes = listOf(object : DelegatingType() {
             override fun getDelegate(): KotlinType {
-                return JvmBuiltIns.Instance.anyType
+                return moduleDescriptor.builtIns.anyType
             }
         })
 
@@ -68,6 +82,74 @@ class BuiltInClassesAreSerializableOnJvm(
         else return listOf()
     }
 
+    override fun getFunctions(name: Name, classDescriptor: DeserializedClassDescriptor): Collection<SimpleFunctionDescriptor> =
+            getAdditionalFunctions(classDescriptor) {
+                it.getContributedFunctions(name, NoLookupLocation.FROM_BUILTINS)
+            }
+            .map {
+                additionalMember ->
+                additionalMember.newCopyBuilder().apply {
+                    setOwner(classDescriptor)
+                    setDispatchReceiverParameter(classDescriptor.thisAsReceiverParameter)
+                    setPreserveSourceElement()
+                    setSubstitution(createMappedTypeParametersSubstitution(
+                            additionalMember.containingDeclaration as ClassDescriptor, classDescriptor))
+                }.build()!!
+            }
+
+    override fun getFunctionsNames(classDescriptor: DeserializedClassDescriptor): Collection<Name> =
+            getAdditionalFunctions(classDescriptor) {
+                it.getContributedDescriptors(DescriptorKindFilter.FUNCTIONS).filterIsInstance<SimpleFunctionDescriptor>()
+            }.map(SimpleFunctionDescriptor::getName)
+
+    private fun getAdditionalFunctions(
+            classDescriptor: DeserializedClassDescriptor,
+            functionsByScope: (MemberScope) -> Collection<SimpleFunctionDescriptor>
+    ): Collection<SimpleFunctionDescriptor> {
+        // Prevents recursive dependency: memberScope(Any) -> memberScope(Object) -> memberScope(Any)
+        // No additional members should be added to Any
+        if (classDescriptor.isAny) return emptyList()
+
+        val fqName = classDescriptor.fqNameUnsafe.check { it.isSafe }?.toSafe() ?: return emptyList()
+
+        val j2kClassMap = JavaToKotlinClassMap.INSTANCE
+        val javaAnalogueFqName = j2kClassMap.mapKotlinToJava(fqName.toUnsafe())?.asSingleFqName() ?: return emptyList()
+
+        if (javaAnalogueFqName in IGNORE_BY_DEFAULT_CLASS_FQ_NAMES) return emptyList()
+
+        val javaAnalogueDescriptor =
+                ownerModuleDescriptor.resolveClassByFqName(javaAnalogueFqName, NoLookupLocation.FROM_BUILTINS) as? LazyJavaClassDescriptor
+                ?: return emptyList()
+
+        val platformClassDescriptors = j2kClassMap.mapPlatformClass(javaAnalogueDescriptor.fqNameSafe, DefaultBuiltIns.Instance)
+        val kotlinMutableClassIfContainer = platformClassDescriptors.lastOrNull() ?: return emptyList()
+        val platformVersions = SmartSet.create(platformClassDescriptors.map { it.fqNameSafe })
+
+        val isMutable = j2kClassMap.isMutable(classDescriptor)
+
+        val fakeJavaClassDescriptor =
+                javaAnalogueDescriptor.copy(
+                        javaResolverCache = JavaResolverCache.EMPTY,
+                        additionalSupertypeClassDescriptor = kotlinMutableClassIfContainer)
+
+        val scope = fakeJavaClassDescriptor.unsubstitutedMemberScope
+
+        return functionsByScope(scope)
+                .filter { analogueMember ->
+                    if (analogueMember.kind != CallableMemberDescriptor.Kind.DECLARATION) return@filter false
+                    if (!analogueMember.visibility.isPublicAPI) return@filter false
+
+                    val methodFqName = analogueMember.fqNameSafe
+
+                    if (methodFqName in BLACK_LIST_METHODS_FQ_NAMES) return@filter false
+                    if ((methodFqName in MUTABLE_METHODS_FQ_NAMES) xor isMutable) return@filter false
+
+                    analogueMember.overriddenDescriptors.none {
+                        it.containingDeclaration.fqNameSafe in platformVersions
+                    }
+                }
+    }
+
     companion object {
         fun isSerializableInJava(classFqName: FqName): Boolean {
             val fqNameUnsafe = classFqName.toUnsafe()
@@ -84,5 +166,35 @@ class BuiltInClassesAreSerializableOnJvm(
             return Serializable::class.java.isAssignableFrom(classViaReflection)
         }
 
+        private val IGNORE_BY_DEFAULT_CLASS_FQ_NAMES =
+                setOf(FqName("java.lang.String")) +
+                JvmPrimitiveType.values().map { it.wrapperFqName }
+
+        private val BLACK_LIST_METHODS_FQ_NAMES =
+                buildPrimitiveValueMethodsSet() +
+                FqName("java.util.Collection.toArray") +
+                FqName("java.util.List.toArray") +
+                FqName("java.util.Set.toArray") +
+                FqName("java.lang.annotation.Annotation.annotationType")
+
+        private val MUTABLE_METHODS_FQ_NAMES =
+                inClass("java.util.Collection",
+                        "removeIf") +
+
+                inClass("java.util.List",
+                        "sort", "replaceAll") +
+
+                inClass("java.util.Map",
+                        "compute", "computeIfAbsent", "computeIfPresent", "remove", "merge", "putIfAbsent", "replace", "replaceAll")
+
+        private fun buildPrimitiveValueMethodsSet() =
+                JvmPrimitiveType.values().mapTo(LinkedHashSet()) {
+                    it.wrapperFqName.child(Name.identifier(it.javaKeywordName + "Value"))
+                }
+
+        private fun inClass(classFqName: String, vararg names: String) =
+                names.mapTo(LinkedHashSet()) { FqName(classFqName).child(Name.identifier(it)) }
     }
-}
+}
+
+private val ClassDescriptor.isAny: Boolean get() = fqNameUnsafe == KotlinBuiltIns.FQ_NAMES.any

core/descriptor.loader.java/src/org/jetbrains/kotlin/load/kotlin/DeserializationComponentsForJava.kt

@@ -43,7 +43,7 @@ class DeserializationComponentsForJava(
                 storageManager, moduleDescriptor, classDataFinder, annotationAndConstantLoader, packageFragmentProvider, localClassResolver,
                 errorReporter, lookupTracker, FlexibleJavaClassifierTypeFactory, ClassDescriptorFactory.EMPTY,
                 notFoundClasses, JavaTypeCapabilitiesLoader,
-                additionalClassPartsProvider = BuiltInClassesAreSerializableOnJvm(moduleDescriptor)
+                additionalClassPartsProvider = BuiltInClassesAreSerializableOnJvm(moduleDescriptor, { moduleDescriptor })
         )
         localClassResolver.setDeserializationComponents(components)
     }

core/descriptor.loader.java/src/org/jetbrains/kotlin/platform/JvmBuiltIns.kt

@@ -16,23 +16,25 @@
 
 package org.jetbrains.kotlin.platform
 
-import org.jetbrains.kotlin.builtins.BuiltInsInitializer
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.load.kotlin.BuiltInClassesAreSerializableOnJvm
 import org.jetbrains.kotlin.serialization.deserialization.AdditionalClassPartsProvider
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.utils.sure
 
-class JvmBuiltIns private constructor() : KotlinBuiltIns() {
-    companion object {
-        private val initializer = BuiltInsInitializer {
-            JvmBuiltIns()
-        }
+class JvmBuiltIns(storageManager: StorageManager) : KotlinBuiltIns(storageManager) {
+    // Module containing JDK classes or having them among dependencies
+    private var ownerModuleDescriptor: ModuleDescriptor? = null
 
-        @JvmStatic
-        val Instance: KotlinBuiltIns
-            get() = initializer.get()
+    fun setOwnerModuleDescriptor(moduleDescriptor: ModuleDescriptor) {
+        assert(ownerModuleDescriptor == null) { "JvmBuiltins repeated initialization" }
+        this.ownerModuleDescriptor = moduleDescriptor
     }
 
     override fun getAdditionalClassPartsProvider(): AdditionalClassPartsProvider {
-        return BuiltInClassesAreSerializableOnJvm(builtInsModule)
+        return BuiltInClassesAreSerializableOnJvm(builtInsModule, {
+            ownerModuleDescriptor.sure { "JvmBuiltins has not been initialized properly" }
+        })
     }
-}
+}

core/descriptor.loader.java/src/org/jetbrains/kotlin/platform/mappingUtil.kt

@@ -0,0 +1,34 @@
+/*
+ * Copyright 2010-2016 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.jetbrains.kotlin.platform
+
+import org.jetbrains.kotlin.descriptors.ClassDescriptor
+import org.jetbrains.kotlin.descriptors.TypeParameterDescriptor
+import org.jetbrains.kotlin.types.TypeConstructorSubstitution
+import org.jetbrains.kotlin.types.typeUtil.asTypeProjection
+
+fun createMappedTypeParametersSubstitution(from: ClassDescriptor, to: ClassDescriptor): TypeConstructorSubstitution {
+    assert(from.declaredTypeParameters.size == to.declaredTypeParameters.size) {
+        "$from and $to should have same number of type parameters, " +
+        "but ${from.declaredTypeParameters.size} / ${to.declaredTypeParameters.size} found"
+    }
+
+    return TypeConstructorSubstitution.createByConstructorsMap(
+            from.declaredTypeParameters.map(TypeParameterDescriptor::getTypeConstructor).zip(
+                    to.declaredTypeParameters.map { it.defaultType.asTypeProjection() }
+            ).toMap())
+}