Make diagnostic for @MetaSerializable on nested annotation

K2 plugin API has a limitation that meta annotations from plugin predicates
can't be used on nested annotations because their resolve process includes supertypes
resolve, that can be affected by the plugin itself. Therefore, @MetaSerializable
can't be applied to nested annotation classes in K2, which is reflected by
this diagnostic.

For old FE, diagnostic is lowered to WARNING with deprecation message.

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationErrors.java

@@ -52,6 +52,7 @@ public interface SerializationErrors {
     DiagnosticFactory3<KtAnnotationEntry, String, String, String> PROVIDED_RUNTIME_TOO_LOW = DiagnosticFactory3.create(ERROR);
 
     DiagnosticFactory2<PsiElement, KotlinType, KotlinType> INCONSISTENT_INHERITABLE_SERIALINFO = DiagnosticFactory2.create(ERROR);
+    DiagnosticFactory0<PsiElement> META_SERIALIZABLE_NOT_APPLICABLE = DiagnosticFactory0.create(WARNING);
 
     DiagnosticFactory2<PsiElement, KotlinType, KotlinType> EXTERNAL_CLASS_NOT_SERIALIZABLE = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory2<PsiElement, KotlinType, KotlinType> EXTERNAL_CLASS_IN_ANOTHER_MODULE = DiagnosticFactory2.create(ERROR);

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginDeclarationChecker.kt

@@ -40,6 +40,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
     final override fun check(declaration: KtDeclaration, descriptor: DeclarationDescriptor, context: DeclarationCheckerContext) {
         if (descriptor !is ClassDescriptor) return
 
+        checkMetaSerializableApplicable(descriptor, context.trace)
         checkEnum(descriptor, declaration, context.trace)
         checkExternalSerializer(descriptor, declaration, context.trace)
 
@@ -60,6 +61,13 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         checkInheritedAnnotations(descriptor, declaration, context.trace)
     }
 
+    private fun checkMetaSerializableApplicable(descriptor: ClassDescriptor, trace: BindingTrace) {
+        if (descriptor.kind != ClassKind.ANNOTATION_CLASS) return
+        if (descriptor.classId?.isNestedClass != true) return
+        val entry = descriptor.findAnnotationDeclaration(SerializationAnnotations.metaSerializableAnnotationFqName) ?: return
+        trace.report(SerializationErrors.META_SERIALIZABLE_NOT_APPLICABLE.on(entry))
+    }
+
     private fun checkExternalSerializer(classDescriptor: ClassDescriptor, declaration: KtDeclaration, trace: BindingTrace) {
         val serializableKType = classDescriptor.serializerForClass ?: return
         val serializableDescriptor = serializableKType.toClassDescriptor ?: return

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginErrorsRendering.kt

@@ -160,6 +160,10 @@ object SerializationPluginErrorsRendering : DefaultErrorMessages.Extension {
             Renderers.RENDER_TYPE,
             Renderers.RENDER_TYPE
         )
+        MAP.put(
+            SerializationErrors.META_SERIALIZABLE_NOT_APPLICABLE,
+            "@MetaSerializable annotation should be used only on top-level annotation classes. Usage on nested annotation classes is deprecated and will yield errors in the future."
+        )
 
         MAP.put(
             SerializationErrors.EXTERNAL_CLASS_NOT_SERIALIZABLE,

plugins/kotlinx-serialization/kotlinx-serialization.k2/src/org/jetbrains/kotlinx/serialization/compiler/fir/checkers/FirSerializationErrors.kt

@@ -45,6 +45,7 @@ object FirSerializationErrors {
     val PROVIDED_RUNTIME_TOO_LOW by error3<KtAnnotationEntry, String, String, String>()
 
     val INCONSISTENT_INHERITABLE_SERIALINFO by error2<PsiElement, ConeKotlinType, ConeKotlinType>()
+    val META_SERIALIZABLE_NOT_APPLICABLE by error0<PsiElement>()
 
     val EXTERNAL_CLASS_NOT_SERIALIZABLE by error2<PsiElement, FirClassSymbol<*>, ConeKotlinType>()
     val EXTERNAL_CLASS_IN_ANOTHER_MODULE by error2<PsiElement, FirClassSymbol<*>, ConeKotlinType>()

plugins/kotlinx-serialization/kotlinx-serialization.k2/src/org/jetbrains/kotlinx/serialization/compiler/fir/checkers/FirSerializationPluginClassChecker.kt

@@ -7,11 +7,13 @@ package org.jetbrains.kotlinx.serialization.compiler.fir.checkers
 
 import org.jetbrains.kotlin.KtSourceElement
 import org.jetbrains.kotlin.config.KotlinCompilerVersion
+import org.jetbrains.kotlin.descriptors.ClassKind
 import org.jetbrains.kotlin.diagnostics.*
 import org.jetbrains.kotlin.fir.FirSession
 import org.jetbrains.kotlin.fir.analysis.checkers.context.CheckerContext
 import org.jetbrains.kotlin.fir.analysis.checkers.declaration.FirClassChecker
 import org.jetbrains.kotlin.fir.analysis.checkers.isInlineClass
+import org.jetbrains.kotlin.fir.analysis.checkers.outerClassSymbol
 import org.jetbrains.kotlin.fir.analysis.checkers.toRegularClassSymbol
 import org.jetbrains.kotlin.fir.declarations.*
 import org.jetbrains.kotlin.fir.declarations.utils.*
@@ -43,6 +45,7 @@ object FirSerializationPluginClassChecker : FirClassChecker() {
     override fun check(declaration: FirClass, context: CheckerContext, reporter: DiagnosticReporter) {
         with(context) {
             val classSymbol = declaration.symbol
+            checkMetaSerializableApplicable(classSymbol, reporter)
             checkEnum(classSymbol, reporter)
             checkExternalSerializer(classSymbol, reporter)
             if (!canBeSerializedInternally(classSymbol, reporter)) return
@@ -58,6 +61,15 @@ object FirSerializationPluginClassChecker : FirClassChecker() {
         }
     }
 
+    context(CheckerContext)
+    private fun checkMetaSerializableApplicable(classSymbol: FirClassSymbol<out FirClass>, reporter: DiagnosticReporter) {
+        if (classSymbol.classKind != ClassKind.ANNOTATION_CLASS) return
+        if (!classSymbol.classId.isNestedClass) return
+        val anno = classSymbol.resolvedAnnotationsWithClassIds.find { it.classId == SerializationAnnotations.metaSerializableAnnotationClassId }
+        if (anno == null) return
+        reporter.reportOn(anno.source, FirSerializationErrors.META_SERIALIZABLE_NOT_APPLICABLE)
+    }
+
     context(CheckerContext)
     @Suppress("IncorrectFormatting") // KTIJ-22227
     private fun checkExternalSerializer(classSymbol: FirClassSymbol<*>, reporter: DiagnosticReporter) {

plugins/kotlinx-serialization/kotlinx-serialization.k2/src/org/jetbrains/kotlinx/serialization/compiler/fir/checkers/KtDefaultErrorMessagesSerialization.kt

@@ -151,6 +151,10 @@ object KtDefaultErrorMessagesSerialization : BaseDiagnosticRendererFactory() {
             FirDiagnosticRenderers.RENDER_TYPE,
             FirDiagnosticRenderers.RENDER_TYPE
         )
+        put(
+            FirSerializationErrors.META_SERIALIZABLE_NOT_APPLICABLE,
+            "@MetaSerializable annotation can be used only on top-level annotation classes."
+        )
 
         put(
             FirSerializationErrors.EXTERNAL_CLASS_NOT_SERIALIZABLE,

plugins/kotlinx-serialization/testData/diagnostics/metaSerializableNested.fir.kt

@@ -0,0 +1,21 @@
+// WITH_STDLIB
+// SKIP_TXT
+
+import kotlinx.serialization.*
+
+@MetaSerializable
+annotation class TopLevel
+
+class MetaSerializableNestedTest {
+    <!META_SERIALIZABLE_NOT_APPLICABLE!>@MetaSerializable<!>
+    @Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
+    annotation class JsonComment(val comment: String)
+
+    object Nested2 {
+        <!META_SERIALIZABLE_NOT_APPLICABLE!>@MetaSerializable<!>
+        annotation class Nested3
+    }
+
+    @<!UNRESOLVED_REFERENCE!>JsonComment<!>("class_comment")
+    data class IntDataCommented(val i: Int)
+}

plugins/kotlinx-serialization/testData/diagnostics/metaSerializableNested.kt

@@ -0,0 +1,21 @@
+// WITH_STDLIB
+// SKIP_TXT
+
+import kotlinx.serialization.*
+
+@MetaSerializable
+annotation class TopLevel
+
+class MetaSerializableNestedTest {
+    <!META_SERIALIZABLE_NOT_APPLICABLE!>@MetaSerializable<!>
+    @Target(AnnotationTarget.PROPERTY, AnnotationTarget.CLASS)
+    annotation class JsonComment(val comment: String)
+
+    object Nested2 {
+        <!META_SERIALIZABLE_NOT_APPLICABLE!>@MetaSerializable<!>
+        annotation class Nested3
+    }
+
+    @JsonComment("class_comment")
+    data class IntDataCommented(val i: Int)
+}

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationFirDiagnosticTestGenerated.java

@@ -92,6 +92,12 @@ public class SerializationFirDiagnosticTestGenerated extends AbstractSerializati
             runTest("plugins/kotlinx-serialization/testData/diagnostics/LocalAndAnonymous.kt");
         }
 
+        @Test
+        @TestMetadata("metaSerializableNested.kt")
+        public void testMetaSerializableNested() throws Exception {
+            runTest("plugins/kotlinx-serialization/testData/diagnostics/metaSerializableNested.kt");
+        }
+
         @Test
         @TestMetadata("NoSuitableCtorInParent.kt")
         public void testNoSuitableCtorInParent() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationPluginDiagnosticTestGenerated.java

@@ -90,6 +90,12 @@ public class SerializationPluginDiagnosticTestGenerated extends AbstractSerializ
         runTest("plugins/kotlinx-serialization/testData/diagnostics/LocalAndAnonymous.kt");
     }
 
+    @Test
+    @TestMetadata("metaSerializableNested.kt")
+    public void testMetaSerializableNested() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/diagnostics/metaSerializableNested.kt");
+    }
+
     @Test
     @TestMetadata("NoSuitableCtorInParent.kt")
     public void testNoSuitableCtorInParent() throws Exception {