[Commonizer] Support Kotlin/Native forward declarations

compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibMetadataDeserializedPackageFragmentsFactoryImpl.kt

@@ -1,8 +1,9 @@
 package org.jetbrains.kotlin.backend.common.serialization.metadata.impl
 
 import org.jetbrains.kotlin.backend.common.serialization.metadata.KlibMetadataDeserializedPackageFragmentsFactory
-import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
-import org.jetbrains.kotlin.descriptors.impl.ClassDescriptorImpl
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.PackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
 import org.jetbrains.kotlin.incremental.components.LookupLocation
 import org.jetbrains.kotlin.incremental.components.NoLookupLocation
@@ -11,15 +12,13 @@ import org.jetbrains.kotlin.library.metadata.KlibMetadataCachedPackageFragment
 import org.jetbrains.kotlin.library.metadata.KlibMetadataDeserializedPackageFragment
 import org.jetbrains.kotlin.library.metadata.KlibMetadataPackageFragment
 import org.jetbrains.kotlin.library.metadata.PackageAccessHandler
+import org.jetbrains.kotlin.library.packageFqName
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
-import org.jetbrains.kotlin.resolve.descriptorUtil.builtIns
 import org.jetbrains.kotlin.resolve.scopes.MemberScope
 import org.jetbrains.kotlin.resolve.scopes.MemberScopeImpl
 import org.jetbrains.kotlin.serialization.konan.impl.ForwardDeclarationsFqNames
-import org.jetbrains.kotlin.storage.LockBasedStorageManager
 import org.jetbrains.kotlin.storage.StorageManager
-import org.jetbrains.kotlin.storage.getValue
 import org.jetbrains.kotlin.utils.Printer
 import org.jetbrains.kotlin.utils.addToStdlib.firstNotNullResult
 
@@ -148,52 +147,3 @@ class ClassifierAliasingPackageFragmentDescriptor(
     override fun getMemberScope(): MemberScope = memberScope
 }
 
-
-/**
- * Package fragment which creates descriptors for forward declarations on demand.
- */
-private class ForwardDeclarationsPackageFragmentDescriptor(
-    storageManager: StorageManager,
-    module: ModuleDescriptor,
-    fqName: FqName,
-    supertypeName: Name,
-    classKind: ClassKind
-) : PackageFragmentDescriptorImpl(module, fqName) {
-
-    private val memberScope = object : MemberScopeImpl() {
-
-        private val declarations = storageManager.createMemoizedFunction(this::createDeclaration)
-
-        private val supertype by storageManager.createLazyValue {
-            val descriptor = builtIns.builtInsModule.getPackage(ForwardDeclarationsFqNames.packageName)
-                .memberScope
-                .getContributedClassifier(supertypeName, NoLookupLocation.FROM_BACKEND) as ClassDescriptor
-
-            descriptor.defaultType
-        }
-
-        private fun createDeclaration(name: Name): ClassDescriptor {
-            return ClassDescriptorImpl(
-                this@ForwardDeclarationsPackageFragmentDescriptor,
-                name,
-                Modality.FINAL,
-                classKind,
-                listOf(supertype),
-                SourceElement.NO_SOURCE,
-                false,
-                LockBasedStorageManager.NO_LOCKS
-            ).apply {
-                this.initialize(MemberScope.Empty, emptySet(), null)
-            }
-        }
-
-        override fun getContributedClassifier(name: Name, location: LookupLocation) = declarations(name)
-
-        override fun printScopeStructure(p: Printer) {
-            p.println(this::class.java.simpleName, "{}")
-        }
-    }
-
-    override fun getMemberScope(): MemberScope = memberScope
-}
-

compiler/util-klib-metadata/src/org/jetbrains/kotlin/library/metadata/impl/KlibResolvedModuleDescriptorsFactoryImpl.kt

@@ -70,7 +70,7 @@ class KlibResolvedModuleDescriptorsFactoryImpl(
         return KotlinResolvedModuleDescriptors(moduleDescriptors, forwardDeclarationsModule)
     }
 
-    private fun createForwardDeclarationsModule(
+    fun createForwardDeclarationsModule(
             builtIns: KotlinBuiltIns?,
             storageManager: StorageManager): ModuleDescriptorImpl {
 
@@ -125,11 +125,11 @@ class KlibResolvedModuleDescriptorsFactoryImpl(
  * Package fragment which creates descriptors for forward declarations on demand.
  */
 class ForwardDeclarationsPackageFragmentDescriptor(
-        storageManager: StorageManager,
+    storageManager: StorageManager,
-        module: ModuleDescriptor,
+    module: ModuleDescriptor,
-        fqName: FqName,
+    fqName: FqName,
-        supertypeName: Name,
+    supertypeName: Name,
-        classKind: ClassKind
+    classKind: ClassKind
 ) : PackageFragmentDescriptorImpl(module, fqName) {
 
     private val memberScope = object : MemberScopeImpl() {
@@ -137,23 +137,23 @@ class ForwardDeclarationsPackageFragmentDescriptor(
         private val declarations = storageManager.createMemoizedFunction(this::createDeclaration)
 
         private val supertype by storageManager.createLazyValue {
-            val descriptor = builtIns.builtInsModule.getPackage(ForwardDeclarationsFqNames.packageName)
+            val descriptor = builtIns.builtInsModule.getPackage(ForwardDeclarationsFqNames.cInterop)
-                    .memberScope
+                .memberScope
-                    .getContributedClassifier(supertypeName, NoLookupLocation.FROM_BACKEND) as ClassDescriptor
+                .getContributedClassifier(supertypeName, NoLookupLocation.FROM_BACKEND) as ClassDescriptor
 
             descriptor.defaultType
         }
 
         private fun createDeclaration(name: Name): ClassDescriptor {
             return ClassDescriptorImpl(
-                    this@ForwardDeclarationsPackageFragmentDescriptor,
+                this@ForwardDeclarationsPackageFragmentDescriptor,
-                    name,
+                name,
-                    Modality.FINAL,
+                Modality.FINAL,
-                    classKind,
+                classKind,
-                    listOf(supertype),
+                listOf(supertype),
-                    SourceElement.NO_SOURCE,
+                SourceElement.NO_SOURCE,
-                    false,
+                false,
-                    LockBasedStorageManager.NO_LOCKS
+                LockBasedStorageManager.NO_LOCKS
             ).apply {
                 this.initialize(MemberScope.Empty, emptySet(), null)
             }
@@ -170,14 +170,16 @@ class ForwardDeclarationsPackageFragmentDescriptor(
 }
 
 // TODO decouple and move interop-specific logic back to Kotlin/Native.
-internal object ForwardDeclarationsFqNames {
+object ForwardDeclarationsFqNames {
 
-    val packageName = FqName("kotlinx.cinterop")
+    internal val cInterop = FqName("kotlinx.cinterop")
 
-    val cNames = FqName("cnames")
+    private val cNames = FqName("cnames")
-    val cNamesStructs = cNames.child(Name.identifier("structs"))
+    internal val cNamesStructs = cNames.child(Name.identifier("structs"))
 
-    val objCNames = FqName("objcnames")
+    private val objCNames = FqName("objcnames")
-    val objCNamesClasses = objCNames.child(Name.identifier("classes"))
+    internal val objCNamesClasses = objCNames.child(Name.identifier("classes"))
-    val objCNamesProtocols = objCNames.child(Name.identifier("protocols"))
+    internal val objCNamesProtocols = objCNames.child(Name.identifier("protocols"))
+
+    val syntheticPackages = setOf(cNames, objCNames)
 }

konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/context.kt

@@ -7,12 +7,16 @@ package org.jetbrains.kotlin.descriptors.commonizer.builder
 
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.commonizer.*
 import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroup
 import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroupMap
 import org.jetbrains.kotlin.descriptors.commonizer.Target
+import org.jetbrains.kotlin.descriptors.commonizer.createKotlinNativeForwardDeclarationsModule
+import org.jetbrains.kotlin.descriptors.commonizer.isUnderKotlinNativeSyntheticPackages
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.dimension
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.indexOfCommon
+import org.jetbrains.kotlin.incremental.components.NoLookupLocation
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.storage.StorageManager
@@ -25,18 +29,24 @@ class DeclarationsBuilderCache(dimension: Int) {
         check(dimension > 0)
     }
 
+    private val modules = CommonizedGroup<Collection<ModuleDescriptor>>(dimension)
     private val packageFragments = CommonizedGroupMap<Pair<Name, FqName>, CommonizedPackageFragmentDescriptor>(dimension)
     private val classes = CommonizedGroupMap<FqName, CommonizedClassDescriptor>(dimension)
     private val typeAliases = CommonizedGroupMap<FqName, CommonizedTypeAliasDescriptor>(dimension)
-    private val modules = CommonizedGroup<Collection<ModuleDescriptor>>(dimension)
+
+    private val forwardDeclarationsModules = CommonizedGroup<ModuleDescriptor>(dimension)
 
     fun getCachedPackageFragments(moduleName: Name, packageFqName: FqName): List<CommonizedPackageFragmentDescriptor?> =
         packageFragments.getOrFail(moduleName to packageFqName)
 
     fun getCachedClasses(fqName: FqName): List<CommonizedClassDescriptor?> = classes.getOrFail(fqName)
 
-    private inline fun <reified K, reified V : DeclarationDescriptor> CommonizedGroupMap<K, V>.getOrFail(key: K): List<V?> =
+    fun getCachedClassifier(fqName: FqName, index: Int): ClassifierDescriptorWithTypeParameters? =
-        getOrNull(key)?.toList() ?: error("No cached ${V::class.java} with key $key found")
+        classes.getOrNull(fqName)?.get(index) ?: typeAliases.getOrNull(fqName)?.get(index)
+
+    fun cache(index: Int, modules: Collection<ModuleDescriptor>) {
+        this.modules[index] = modules
+    }
 
     fun cache(moduleName: Name, packageFqName: FqName, index: Int, descriptor: CommonizedPackageFragmentDescriptor) {
         packageFragments[moduleName to packageFqName][index] = descriptor
@@ -50,14 +60,30 @@ class DeclarationsBuilderCache(dimension: Int) {
         typeAliases[fqName][index] = descriptor
     }
 
-    fun cache(index: Int, modules: Collection<ModuleDescriptor>) {
+    fun computeIfAbsentForwardDeclarationsModule(index: Int, computable: () -> ModuleDescriptor): ModuleDescriptor {
-        this.modules[index] = modules
+        forwardDeclarationsModules[index]?.let { return it }
+
+        val module = computable()
+        forwardDeclarationsModules[index] = module
+        return module
     }
 
-    fun getCachedClassifier(fqName: FqName, index: Int): ClassifierDescriptorWithTypeParameters? =
+    fun getAllModules(index: Int): Collection<ModuleDescriptor> {
-        classes.getOrNull(fqName)?.get(index) ?: typeAliases.getOrNull(fqName)?.get(index)
+        val modulesForTarget = modules[index] ?: error("No module descriptors found for index $index")
+        val forwardDeclarationsModule = forwardDeclarationsModules[index]
 
-    fun getCachedModules(index: Int): Collection<ModuleDescriptor> = modules[index] ?: error("No module descriptors found for index $index")
+        // forward declarations module is created on demand (and only when commonizing Kotlin/Native target)
+        // so, don't return it if it's not necessary
+        return if (forwardDeclarationsModule != null)
+            modulesForTarget + forwardDeclarationsModule
+        else
+            modulesForTarget
+    }
+
+    companion object {
+        private inline fun <reified K, reified V : DeclarationDescriptor> CommonizedGroupMap<K, V>.getOrFail(key: K): List<V?> =
+            getOrNull(key)?.toList() ?: error("No cached ${V::class.java} with key $key found")
+    }
 }
 
 class GlobalDeclarationsBuilderComponents(
@@ -79,8 +105,34 @@ class TargetDeclarationsBuilderComponents(
     val index: Int,
     private val cache: DeclarationsBuilderCache
 ) {
-    // only for classes and type aliases
+    // N.B. this function may create new classifiers for types from Kotlin/Native forward declarations packages
-    fun getCachedClassifier(fqName: FqName): ClassifierDescriptorWithTypeParameters? = cache.getCachedClassifier(fqName, index)
+    fun findAppropriateClassOrTypeAlias(fqName: FqName): ClassifierDescriptorWithTypeParameters? {
+
+        return if (fqName.isUnderKotlinNativeSyntheticPackages) {
+            // that's a synthetic Kotlin/Native classifier that was exported as forward declaration in one or more modules,
+            // but did not match any existing class or typealias
+            val module = cache.computeIfAbsentForwardDeclarationsModule(index) {
+                // N.B. forward declarations module is created only on demand
+                createKotlinNativeForwardDeclarationsModule(
+                    storageManager = storageManager,
+                    builtIns = builtIns
+                )
+            }
+
+            // create and return new classifier
+            module.packageFragmentProvider
+                .getPackageFragments(fqName.parent())
+                .single()
+                .getMemberScope()
+                .getContributedClassifier(
+                    name = fqName.shortName(),
+                    location = NoLookupLocation.FOR_ALREADY_TRACKED
+                ) as ClassifierDescriptorWithTypeParameters
+        } else {
+            // look up in created descriptors cache
+            cache.getCachedClassifier(fqName, index)
+        }
+    }
 }
 
 fun CirRootNode.createGlobalBuilderComponents(storageManager: StorageManager): GlobalDeclarationsBuilderComponents {

konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/builder/utils.kt

@@ -136,19 +136,22 @@ internal fun CirSimpleType.buildType(
 internal fun findClassOrTypeAlias(
     targetComponents: TargetDeclarationsBuilderComponents,
     fqName: FqName
-): ClassifierDescriptorWithTypeParameters {
+): ClassifierDescriptorWithTypeParameters = when {
-    return if (fqName.isUnderStandardKotlinPackages) {
+    fqName.isUnderStandardKotlinPackages -> {
         // look up for classifier in built-ins module:
         val builtInsModule = targetComponents.builtIns.builtInsModule
+
         // TODO: this works fine for Native as far as built-ins module contains full Native stdlib, but this is not enough for JVM and JS
         builtInsModule.getPackage(fqName.parent())
             .memberScope
             .getContributedClassifier(fqName.shortName(), NoLookupLocation.FOR_ALREADY_TRACKED)
             ?.cast()
             ?: error("Classifier $fqName not found in built-ins module $builtInsModule")
-    } else {
+    }
-        // otherwise, look up in created descriptors cache:
+
-        targetComponents.getCachedClassifier(fqName)
+    else -> {
+        // otherwise, find the appropriate user classifier:
+        targetComponents.findAppropriateClassOrTypeAlias(fqName)
             ?: error("Classifier $fqName not found in created descriptors cache")
     }
 }

konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/facade.kt

@@ -83,7 +83,7 @@ fun runCommonization(parameters: CommonizationParameters): CommonizationResult {
         val target = it.target
         check(target !in modulesByTargets)
 
-        modulesByTargets[target] = components.cache.getCachedModules(it.index)
+        modulesByTargets[target] = components.cache.getAllModules(it.index)
     }
 
     return CommonizationPerformed(modulesByTargets)

konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/mergedtree/modules.kt

@@ -5,14 +5,15 @@
 
 package org.jetbrains.kotlin.descriptors.commonizer.mergedtree
 
+import org.jetbrains.kotlin.backend.common.serialization.metadata.impl.ExportedForwardDeclarationsPackageFragmentDescriptor
 import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.commonizer.*
 import org.jetbrains.kotlin.descriptors.commonizer.CommonizedGroupMap
 import org.jetbrains.kotlin.descriptors.commonizer.firstNonNull
+import org.jetbrains.kotlin.descriptors.commonizer.isUnderKotlinNativeSyntheticPackages
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirModuleNode
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.CirRootNode
 import org.jetbrains.kotlin.descriptors.commonizer.mergedtree.ir.buildModuleNode
-import org.jetbrains.kotlin.descriptors.commonizer.toList
-import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.resolve.scopes.ChainedMemberScope
 import org.jetbrains.kotlin.resolve.scopes.MemberScope
@@ -46,11 +47,15 @@ internal fun mergeModules(
 internal fun ModuleDescriptor.collectNonEmptyPackageMemberScopes(collector: (FqName, MemberScope) -> Unit) {
     // we don's need to process fragments from other modules which are the dependencies of this module, so
     // let's use the appropriate package fragment provider
-    val packageFragmentProvider = (this as ModuleDescriptorImpl).packageFragmentProviderForModuleContentWithoutDependencies
+    val packageFragmentProvider = this.packageFragmentProvider
 
     fun recurse(packageFqName: FqName) {
+        if (packageFqName.isUnderKotlinNativeSyntheticPackages)
+            return
+
         val ownPackageFragments = packageFragmentProvider.getPackageFragments(packageFqName)
         val ownPackageMemberScopes = ownPackageFragments.asSequence()
+            .filter { it !is ExportedForwardDeclarationsPackageFragmentDescriptor }
             .map { it.getMemberScope() }
             .filter { it != MemberScope.Empty }
             .toList(ownPackageFragments.size)

konan/commonizer/src/org/jetbrains/kotlin/descriptors/commonizer/utils.kt

@@ -7,13 +7,18 @@ package org.jetbrains.kotlin.descriptors.commonizer
 
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.descriptors.ClassifierDescriptor
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
+import org.jetbrains.kotlin.descriptors.impl.ModuleDescriptorImpl
+import org.jetbrains.kotlin.konan.library.KonanFactories.DefaultDeserializedDescriptorFactory
+import org.jetbrains.kotlin.konan.library.KonanFactories.createDefaultKonanResolvedModuleDescriptorsFactory
 import org.jetbrains.kotlin.name.FqName
-import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
+import org.jetbrains.kotlin.serialization.konan.impl.ForwardDeclarationsFqNames
+import org.jetbrains.kotlin.serialization.konan.impl.KlibResolvedModuleDescriptorsFactoryImpl
+import org.jetbrains.kotlin.storage.StorageManager
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.TypeUtils.getTypeParameterDescriptorOrNull
 import org.jetbrains.kotlin.types.Variance
-import org.jetbrains.kotlin.types.isNullable
 import org.jetbrains.kotlin.types.typeUtil.makeNotNullable
 import org.jetbrains.kotlin.utils.addToStdlib.firstIsInstance
 
@@ -79,7 +84,44 @@ private fun StringBuilder.buildFqNameWithTypeParameters(type: KotlinType, explor
 
 internal fun Any?.isNull(): Boolean = this == null
 
-private val KOTLINX_PACKAGE_NAME = Name.identifier("kotlinx")
+private val STANDARD_KOTLIN_PACKAGE_PREFIXES = listOf(
+    KotlinBuiltIns.BUILT_INS_PACKAGE_NAME.asString(),
+    "kotlinx"
+)
+
+private val KOTLIN_NATIVE_SYNTHETIC_PACKAGES_PREFIXES = ForwardDeclarationsFqNames.syntheticPackages
+    .map { fqName ->
+        check(!fqName.isRoot)
+        fqName.asString()
+    }
 
 internal val FqName.isUnderStandardKotlinPackages: Boolean
-    get() = startsWith(KotlinBuiltIns.BUILT_INS_PACKAGE_NAME) || startsWith(KOTLINX_PACKAGE_NAME)
+    get() = hasAnyPrefix(STANDARD_KOTLIN_PACKAGE_PREFIXES)
+
+internal val FqName.isUnderKotlinNativeSyntheticPackages: Boolean
+    get() = hasAnyPrefix(KOTLIN_NATIVE_SYNTHETIC_PACKAGES_PREFIXES)
+
+private fun FqName.hasAnyPrefix(prefixes: List<String>): Boolean =
+    asString().let { fqName ->
+        prefixes.any { prefix ->
+            val lengthDifference = fqName.length - prefix.length
+            when {
+                lengthDifference == 0 -> fqName == prefix
+                lengthDifference > 0 -> fqName[prefix.length] == '.' && fqName.startsWith(prefix)
+                else -> false
+            }
+        }
+    }
+
+internal val ModuleDescriptor.packageFragmentProvider
+    get() = (this as ModuleDescriptorImpl).packageFragmentProviderForModuleContentWithoutDependencies
+
+internal fun createKotlinNativeForwardDeclarationsModule(
+    storageManager: StorageManager,
+    builtIns: KotlinBuiltIns
+) =
+    (createDefaultKonanResolvedModuleDescriptorsFactory(DefaultDeserializedDescriptorFactory) as KlibResolvedModuleDescriptorsFactoryImpl)
+        .createForwardDeclarationsModule(
+            builtIns = builtIns,
+            storageManager = storageManager
+        )