Prohibit serializable inner classes

Fixes https://github.com/Kotlin/kotlinx.serialization/issues/1101

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationErrors.java

@@ -17,6 +17,7 @@ public interface SerializationErrors {
     DiagnosticFactory2<PsiElement, String, String> INLINE_CLASSES_NOT_SUPPORTED = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory0<PsiElement> PLUGIN_IS_NOT_ENABLED = DiagnosticFactory0.create(WARNING);
     DiagnosticFactory0<PsiElement> LOCAL_CLASSES_NOT_SUPPORTED = DiagnosticFactory0.create(ERROR);
+    DiagnosticFactory0<PsiElement> INNER_CLASSES_NOT_SUPPORTED = DiagnosticFactory0.create(ERROR);
 
     DiagnosticFactory0<PsiElement> EXPLICIT_SERIALIZABLE_IS_REQUIRED = DiagnosticFactory0.create(WARNING);
 

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginDeclarationChecker.kt

@@ -123,6 +123,11 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             return false
         }
 
+        if (descriptor.isInner) {
+            trace.reportOnSerializableAnnotation(descriptor, SerializationErrors.INNER_CLASSES_NOT_SUPPORTED)
+            return false
+        }
+
         if (descriptor.isInlineClass() && !canSupportInlineClasses(descriptor.module, trace)) {
             descriptor.onSerializableAnnotation {
                 trace.report(

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginErrorsRendering.kt

@@ -27,7 +27,11 @@ object SerializationPluginErrorsRendering : DefaultErrorMessages.Extension {
         )
         MAP.put(
             SerializationErrors.LOCAL_CLASSES_NOT_SUPPORTED,
-            "Local class and anonymous objects can not be serializable"
+            "Local classes and anonymous objects can not be serializable."
+        )
+        MAP.put(
+            SerializationErrors.INNER_CLASSES_NOT_SUPPORTED,
+            "Inner (with reference to outer this) serializable classes are not supported. Remove @Serializable annotation or 'inner' keyword."
         )
         MAP.put(
             SerializationErrors.EXPLICIT_SERIALIZABLE_IS_REQUIRED,

plugins/kotlin-serialization/kotlin-serialization-compiler/testData/diagnostics/LocalAndAnonymous.kt

@@ -14,5 +14,9 @@ fun container() {
 val topLevelAnon = <!LOCAL_CLASSES_NOT_SUPPORTED!>@Serializable<!> object {}
 
 @Serializable class A {
-    @Serializable class B // nesting classes are allowed
+    @Serializable class B // nested classes are allowed
+
+    <!INNER_CLASSES_NOT_SUPPORTED!>@Serializable<!> inner class C // inner classes are not
+
+    @Serializable object F {} // regular named object, OK
 }