Support new scheme of compilation of OptionalExpectation annotations

Instead of generating these annotation classes as package-private on
JVM, serialize their metadata to the .kotlin_module file, and load it
when compiling dependent multiplatform modules.

The problem with generating them as package-private was that
kotlin-stdlib for JVM would end up declaring symbols from other
platforms, which would include some annotations from package
kotlin.native. But using that package is discouraged by some tools
because it has a Java keyword in its name. In particular, jlink refused
to work with such artifact altogether (KT-21266).

 #KT-38652 Fixed

compiler/frontend.java/src/org/jetbrains/kotlin/frontend/java/di/injection.kt

@@ -44,6 +44,7 @@ import org.jetbrains.kotlin.platform.TargetPlatform
 import org.jetbrains.kotlin.resolve.*
 import org.jetbrains.kotlin.resolve.jvm.JavaDescriptorResolver
 import org.jetbrains.kotlin.resolve.jvm.JvmDiagnosticComponents
+import org.jetbrains.kotlin.resolve.jvm.multiplatform.OptionalAnnotationPackageFragmentProvider
 import org.jetbrains.kotlin.resolve.jvm.platform.JvmPlatformAnalyzerServices
 import org.jetbrains.kotlin.resolve.lazy.KotlinCodeAnalyzer
 import org.jetbrains.kotlin.resolve.lazy.declarations.DeclarationProviderFactory
@@ -121,6 +122,7 @@ fun StorageComponentContainer.configureJavaSpecificComponents(
         useInstance((moduleContext.module.builtIns as JvmBuiltIns).settings)
         useImpl<JvmBuiltInsPackageFragmentProvider>()
     }
+    useImpl<OptionalAnnotationPackageFragmentProvider>()
 
     useInstance(javaClassTracker ?: JavaClassesTracker.Default)
     useInstance(

compiler/frontend.java/src/org/jetbrains/kotlin/load/kotlin/incremental/IncrementalPackagePartProvider.kt

@@ -16,18 +16,20 @@
 
 package org.jetbrains.kotlin.load.kotlin.incremental
 
+import org.jetbrains.kotlin.load.kotlin.JvmPackagePartProviderBase
 import org.jetbrains.kotlin.load.kotlin.PackagePartProvider
 import org.jetbrains.kotlin.load.kotlin.incremental.components.IncrementalCache
 import org.jetbrains.kotlin.load.kotlin.loadModuleMapping
 import org.jetbrains.kotlin.metadata.jvm.deserialization.ModuleMapping
 import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.serialization.deserialization.ClassData
 import org.jetbrains.kotlin.serialization.deserialization.DeserializationConfiguration
 import org.jetbrains.kotlin.storage.StorageManager
 
 class IncrementalPackagePartProvider(
-        private val parent: PackagePartProvider,
-        incrementalCaches: List<IncrementalCache>,
-        storageManager: StorageManager
+    private val parent: PackagePartProvider,
+    incrementalCaches: List<IncrementalCache>,
+    storageManager: StorageManager
 ) : PackagePartProvider {
     lateinit var deserializationConfiguration: DeserializationConfiguration
 
@@ -48,4 +50,8 @@ class IncrementalPackagePartProvider(
     override fun getAnnotationsOnBinaryModule(moduleName: String): List<ClassId> {
         return parent.getAnnotationsOnBinaryModule(moduleName)
     }
+
+    override fun getAllOptionalAnnotationClasses(): List<ClassData> =
+        moduleMappings().flatMap((JvmPackagePartProviderBase)::getAllOptionalAnnotationClasses) +
+                parent.getAllOptionalAnnotationClasses()
 }

compiler/frontend.java/src/org/jetbrains/kotlin/resolve/jvm/multiplatform/OptionalAnnotationPackageFragmentProvider.kt

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2010-2019 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.resolve.jvm.multiplatform
+
+import org.jetbrains.kotlin.config.LanguageVersionSettings
+import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.descriptors.impl.PackageFragmentDescriptorImpl
+import org.jetbrains.kotlin.incremental.components.LookupLocation
+import org.jetbrains.kotlin.incremental.components.LookupTracker
+import org.jetbrains.kotlin.load.kotlin.PackagePartProvider
+import org.jetbrains.kotlin.name.ClassId
+import org.jetbrains.kotlin.name.FqName
+import org.jetbrains.kotlin.name.Name
+import org.jetbrains.kotlin.resolve.CompilerDeserializationConfiguration
+import org.jetbrains.kotlin.resolve.sam.SamConversionResolver
+import org.jetbrains.kotlin.resolve.scopes.DescriptorKindFilter
+import org.jetbrains.kotlin.resolve.scopes.MemberScope
+import org.jetbrains.kotlin.resolve.scopes.MemberScopeImpl
+import org.jetbrains.kotlin.serialization.deserialization.*
+import org.jetbrains.kotlin.serialization.deserialization.builtins.BuiltInSerializerProtocol
+import org.jetbrains.kotlin.storage.NotNullLazyValue
+import org.jetbrains.kotlin.storage.StorageManager
+import org.jetbrains.kotlin.storage.getValue
+import org.jetbrains.kotlin.utils.Printer
+
+class OptionalAnnotationPackageFragmentProvider(
+    module: ModuleDescriptor,
+    storageManager: StorageManager,
+    notFoundClasses: NotFoundClasses,
+    languageVersionSettings: LanguageVersionSettings,
+    packagePartProvider: PackagePartProvider,
+) : PackageFragmentProvider {
+    val packages: Map<FqName, PackageFragmentDescriptor> by storageManager.createLazyValue p@{
+        // We call getAllOptionalAnnotationClasses under lazy value only because IncrementalPackagePartProvider requires
+        // deserializationConfiguration to be injected.
+        val optionalAnnotationClasses = packagePartProvider.getAllOptionalAnnotationClasses()
+        if (optionalAnnotationClasses.isEmpty()) return@p emptyMap()
+
+        mutableMapOf<FqName, PackageFragmentDescriptor>().also { packages ->
+            // We use BuiltInSerializerProtocol when serializing optional annotation classes (see
+            // JvmOptionalAnnotationSerializerExtension). Use it in deserialization as well, to be able to load annotations on
+            // optional annotation classes and their members (in particular, the `@OptionalExpectation` annotation).
+            val serializerProtocol = BuiltInSerializerProtocol
+
+            val classDataFinder = OptionalAnnotationClassDataFinder(optionalAnnotationClasses)
+            val components = storageManager.createLazyValue {
+                DeserializationComponents(
+                    storageManager, module, CompilerDeserializationConfiguration(languageVersionSettings),
+                    classDataFinder,
+                    AnnotationAndConstantLoaderImpl(module, notFoundClasses, serializerProtocol),
+                    this,
+                    LocalClassifierTypeSettings.Default,
+                    ErrorReporter.DO_NOTHING,
+                    LookupTracker.DO_NOTHING,
+                    FlexibleTypeDeserializer.ThrowException,
+                    emptyList(),
+                    notFoundClasses,
+                    ContractDeserializer.DEFAULT,
+                    extensionRegistryLite = serializerProtocol.extensionRegistry,
+                    samConversionResolver = SamConversionResolver.Empty
+                )
+            }
+
+            for ((packageFqName, classes) in classDataFinder.classIdToData.entries.groupBy { it.key.packageFqName }) {
+                val classNames = classes.mapNotNull { (classId) ->
+                    classId.shortClassName.takeUnless { classId.isNestedClass }
+                }.toSet()
+                // TODO: make this lazy value more granular, e.g. a memoized function ClassId -> ClassDescriptor
+                val classDescriptors = storageManager.createLazyValue {
+                    classes.mapNotNull { (classId, classData) ->
+                        components().classDeserializer.deserializeClass(classId, classData)
+                    }.associateBy(ClassDescriptor::getName)
+                }
+                packages[packageFqName] = PackageFragmentForOptionalAnnotations(module, packageFqName, classNames, classDescriptors)
+            }
+        }
+    }
+
+    override fun getPackageFragments(fqName: FqName): List<PackageFragmentDescriptor> =
+        packages[fqName]?.let(::listOf).orEmpty()
+
+    override fun getSubPackagesOf(fqName: FqName, nameFilter: (Name) -> Boolean): Collection<FqName> =
+        emptyList()
+}
+
+private class OptionalAnnotationClassDataFinder(classes: List<ClassData>) : ClassDataFinder {
+    val classIdToData = classes.associateBy { (nameResolver, klass) -> nameResolver.getClassId(klass.fqName) }
+
+    override fun findClassData(classId: ClassId): ClassData? = classIdToData[classId]
+}
+
+private class PackageFragmentForOptionalAnnotations(
+    module: ModuleDescriptor,
+    fqName: FqName,
+    classNames: Set<Name>,
+    classDescriptors: NotNullLazyValue<Map<Name, ClassDescriptor>>,
+) : PackageFragmentDescriptorImpl(module, fqName) {
+    private val scope = object : MemberScopeImpl() {
+        override fun getContributedClassifier(name: Name, location: LookupLocation): ClassifierDescriptor? = classDescriptors()[name]
+
+        override fun getContributedDescriptors(
+            kindFilter: DescriptorKindFilter,
+            nameFilter: (Name) -> Boolean
+        ): Collection<DeclarationDescriptor> =
+            if (kindFilter.acceptsKinds(DescriptorKindFilter.CLASSIFIERS_MASK)) classDescriptors().values else emptyList()
+
+        override fun getClassifierNames(): Set<Name> = classNames
+
+        override fun printScopeStructure(p: Printer) {
+            p.print("PackageFragmentForOptionalAnnotations{${classNames.joinToString(transform = Name::asString)}}")
+        }
+    }
+
+    override fun getMemberScope(): MemberScope = scope
+}