[KxSerialization] Added inspection on abstract custom serializer

Serialization requires an instance of the serializer, which cannot be obtained with the passed interface, abstract or sealed class.
Therefore, the specifying of such classes in `Serializable` annotation must be prohibited.

Fixes https://github.com/Kotlin/kotlinx.serialization/issues/2173
Relates #KT-58036

Merge-request: KT-MR-10753
Merged-by: Sergey Shanshin <Sergey.Shanshin@jetbrains.com>

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationErrors.java

@@ -36,6 +36,7 @@ public interface SerializationErrors {
     DiagnosticFactory3<PsiElement, KotlinType, String, String> DUPLICATE_SERIAL_NAME_ENUM = DiagnosticFactory3.create(ERROR);
     DiagnosticFactory1<PsiElement, KotlinType> SERIALIZER_NOT_FOUND = DiagnosticFactory1.create(ERROR);
     DiagnosticFactory2<PsiElement, KotlinType, KotlinType> SERIALIZER_NULLABILITY_INCOMPATIBLE = DiagnosticFactory2.create(ERROR);
+    DiagnosticFactory2<PsiElement, KotlinType, KotlinType> ABSTRACT_SERIALIZER_TYPE = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory3<PsiElement, KotlinType, KotlinType, KotlinType> SERIALIZER_TYPE_INCOMPATIBLE = DiagnosticFactory3.create(WARNING);
     DiagnosticFactory1<PsiElement, KotlinType> LOCAL_SERIALIZER_USAGE = DiagnosticFactory1.create(ERROR);
     DiagnosticFactory0<PsiElement> TRANSIENT_MISSING_INITIALIZER = DiagnosticFactory0.create(ERROR);

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginDeclarationChecker.kt

@@ -325,6 +325,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         val annotationPsi = descriptor.findSerializableOrMetaAnnotationDeclaration()
         checkCustomSerializerMatch(descriptor.module, descriptor.defaultType, descriptor, annotationPsi, trace, declaration)
         checkCustomSerializerIsNotLocal(descriptor.module, descriptor, trace, declaration)
+        checkCustomSerializerNotAbstract(descriptor.module, descriptor.defaultType, descriptor, annotationPsi, trace, declaration)
     }
 
     private val ClassDescriptor.isAnonymousObjectOrContained: Boolean
@@ -440,6 +441,14 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             if (serializer != null) {
                 val element = ktType?.typeElement
                 checkCustomSerializerMatch(it.module, it.type, it.descriptor, element, trace, propertyPsi)
+                checkCustomSerializerNotAbstract(
+                    it.module,
+                    it.type,
+                    it.descriptor,
+                    it.descriptor.findSerializableOrMetaAnnotationDeclaration(),
+                    trace,
+                    propertyPsi
+                )
                 checkCustomSerializerIsNotLocal(it.module, it.descriptor, trace, propertyPsi)
                 checkSerializerNullability(it.type, serializer.defaultType, element, trace, propertyPsi)
                 generatorContextForAnalysis.checkTypeArguments(it.module, it.type, element, trace, propertyPsi)
@@ -519,6 +528,26 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         }
     }
 
+    private fun checkCustomSerializerNotAbstract(
+        module: ModuleDescriptor,
+        classType: KotlinType,
+        descriptor: Annotated,
+        element: KtElement?,
+        trace: BindingTrace,
+        fallbackElement: PsiElement
+    ) {
+        val serializerType = descriptor.annotations.serializableWith(module) ?: return
+        if (serializerType.toClassDescriptor?.isAbstractOrSealedOrInterface == true) {
+            trace.report(
+                SerializationErrors.ABSTRACT_SERIALIZER_TYPE.on(
+                    element ?: fallbackElement,
+                    classType,
+                    serializerType
+                )
+            )
+        }
+    }
+
     private fun checkCustomSerializerMatch(
         module: ModuleDescriptor,
         classType: KotlinType,

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginErrorsRendering.kt

@@ -107,6 +107,12 @@ object SerializationPluginErrorsRendering : DefaultErrorMessages.Extension {
             Renderers.RENDER_TYPE,
             Renderers.RENDER_TYPE
         )
+        MAP.put(
+            SerializationErrors.ABSTRACT_SERIALIZER_TYPE,
+            "Custom serializer ''{1}'' on serializable type ''{0}'' can not be instantiated. It is not allowed to specify the interface, abstract or sealed class as a custom serializer.",
+            Renderers.RENDER_TYPE,
+            Renderers.RENDER_TYPE
+        )
         MAP.put(
             SerializationErrors.LOCAL_SERIALIZER_USAGE,
             "Class ''{0}'' can't be used as a serializer since it is local",

plugins/kotlinx-serialization/kotlinx-serialization.k1/src/org/jetbrains/kotlinx/serialization/compiler/resolve/KSerializationUtil.kt

@@ -124,6 +124,9 @@ val ClassDescriptor.isInternallySerializableObject: Boolean
 val ClassDescriptor.isSealedSerializableInterface: Boolean
     get() = kind == ClassKind.INTERFACE && modality == Modality.SEALED && hasSerializableOrMetaAnnotation
 
+val ClassDescriptor.isAbstractOrSealedOrInterface: Boolean
+    get() = kind == ClassKind.INTERFACE || modality == Modality.SEALED || modality == Modality.ABSTRACT
+
 val ClassDescriptor.isInternalSerializable: Boolean //todo normal checking
     get() {
         if (kind != ClassKind.CLASS) return false