Added support serialization of local classes in IR backend

Resolves Kotlin/kotlinx.serialization#1427

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/GeneratorHelpers.kt

@@ -717,6 +717,18 @@ interface IrBuilderExtension {
         )
     }
 
+    fun createClassReference(irClass: IrClass, startOffset: Int, endOffset: Int): IrClassReference {
+        val classType = irClass.descriptor.toSimpleType(false)
+        val classSymbol = irClass.symbol
+        return IrClassReferenceImpl(
+            startOffset,
+            endOffset,
+            compilerContext.irBuiltIns.kClassClass.starProjectedType,
+            classSymbol,
+            classType.approximateJvmErasure.toIrType()
+        )
+    }
+
     // It is impossible to use star projections right away, because Array<Int>::class differ from Array<String>::class :
     // detailed information about type arguments required for class references on jvm
     private val KotlinType.approximateJvmErasure: KotlinType

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializableCompanionIrGenerator.kt

@@ -20,6 +20,7 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.calls.components.isVararg
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
 import org.jetbrains.kotlin.resolve.descriptorUtil.module
+import org.jetbrains.kotlin.util.collectionUtils.filterIsInstanceAnd
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.SerializableCompanionCodegen
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.findTypeSerializer
 import org.jetbrains.kotlinx.serialization.compiler.extensions.SerializationPluginContext
@@ -58,7 +59,7 @@ class SerializableCompanionIrGenerator(
             )
         ) ?: return
 
-        val irSerializableClass = compilerContext.referenceClass(serializableDescriptor.fqNameSafe)?.owner ?: return
+        val irSerializableClass = if (irClass.isCompanion) irClass.parentAsClass else irClass
         val serializableWithAlreadyPresent = irSerializableClass.annotations.any {
             it.symbol.descriptor.constructedClass.fqNameSafe == annotationMarkerClass.fqNameSafe
         }
@@ -66,12 +67,23 @@ class SerializableCompanionIrGenerator(
 
         val annotationCtor = compilerContext.referenceConstructors(annotationMarkerClass.fqNameSafe).single { it.owner.isPrimary }
         val annotationType = annotationMarkerClass.defaultType.toIrType()
+
+        val serializerIrClass = if (serializableDescriptor.isInternalSerializable) {
+            // internally generated serializer always declared inside serializable class
+            irSerializableClass.declarations
+                .filterIsInstanceAnd<IrClass> { it.name == serializer.name }
+                .singleOrNull() ?: throw Exception("No class with name ${serializer.fqNameSafe}")
+        } else {
+            // FIXME referenceClass not supports local classes so it should be replaced in future
+            compilerContext.referenceClass(serializer.fqNameSafe)!!.owner
+        }
+
+
         val annotationCtorCall = IrConstructorCallImpl.fromSymbolDescriptor(startOffset, endOffset, annotationType, annotationCtor).apply {
-            val serializerType = serializer.toSimpleType(false)
             putValueArgument(
                 0,
                 createClassReference(
-                    serializerType,
+                    serializerIrClass,
                     startOffset,
                     endOffset
                 )

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializableIrGenerator.kt

@@ -22,6 +22,7 @@ import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
 import org.jetbrains.kotlin.resolve.descriptorUtil.module
 import org.jetbrains.kotlin.util.OperatorNameConventions
+import org.jetbrains.kotlin.util.collectionUtils.filterIsInstanceAnd
 import org.jetbrains.kotlin.utils.getOrPutNullable
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.*
 import org.jetbrains.kotlinx.serialization.compiler.diagnostic.serializableAnnotationIsUseless
@@ -172,10 +173,15 @@ class SerializableIrGenerator(
 
     private fun IrBlockBodyBuilder.getStaticSerialDescriptorExpr(): IrExpression {
         val serializer = serializableDescriptor.classSerializer!!
-        val serialDescriptorGetter = compilerContext.referenceClass(serializer.fqNameSafe)?.getPropertyGetter(SERIAL_DESC_FIELD)
-            ?: throw Exception("No class with name ${serializer.fqNameSafe}")
+        // internally generated serializer always declared inside serializable class
+        val serializerIrClass = irClass.declarations
+            .filterIsInstanceAnd<IrClass> { it.name == serializer.name }
+            .singleOrNull() ?: throw Exception("No class with name ${serializer.fqNameSafe}")
+
+        val serialDescriptorGetter =
+            serializerIrClass.getPropertyGetter(SERIAL_DESC_FIELD)!!
         return irGet(
-            serialDescriptorGetter.owner.returnType,
+            serializerIrClass.defaultType,
             irGetObject(serializer),
             serialDescriptorGetter.owner.symbol
         )

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializerIrGenerator.kt

@@ -10,6 +10,7 @@ import org.jetbrains.kotlin.backend.common.lower.DeclarationIrBuilder
 import org.jetbrains.kotlin.backend.common.lower.irIfThen
 import org.jetbrains.kotlin.backend.common.lower.irThrow
 import org.jetbrains.kotlin.builtins.PrimitiveType
+import org.jetbrains.kotlin.codegen.CompilationException
 import org.jetbrains.kotlin.descriptors.*
 import org.jetbrains.kotlin.ir.builders.*
 import org.jetbrains.kotlin.ir.declarations.*
@@ -22,6 +23,7 @@ import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.platform.jvm.isJvm
 import org.jetbrains.kotlin.resolve.BindingContext
+import org.jetbrains.kotlin.resolve.DescriptorUtils
 import org.jetbrains.kotlin.resolve.calls.components.hasDefaultValue
 import org.jetbrains.kotlin.resolve.descriptorUtil.fqNameSafe
 import org.jetbrains.kotlin.resolve.isInlineClass
@@ -458,6 +460,15 @@ open class SerializerIrGenerator(
             val ctor: IrConstructorSymbol = serializableSyntheticConstructor(serializableIrClass)
             +irReturn(irInvoke(null, ctor, typeArgs, args))
         } else {
+            if (DescriptorUtils.isLocal(serializerDescriptor)) {
+                // if the serializer is local, then the serializable class too, since they must be in the same scope
+                throw CompilationException(
+                    "External serializer class `${serializerDescriptor.fqNameSafe}` is local. Local external serializers are not supported yet.",
+                    null,
+                    null
+                )
+            }
+
             generateGoldenMaskCheck(bitMasks, properties, localSerialDesc.get())
 
             val ctor: IrConstructorSymbol =

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationErrors.java

@@ -16,7 +16,7 @@ import static org.jetbrains.kotlin.diagnostics.Severity.WARNING;
 public interface SerializationErrors {
     DiagnosticFactory2<PsiElement, String, String> INLINE_CLASSES_NOT_SUPPORTED = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory0<PsiElement> PLUGIN_IS_NOT_ENABLED = DiagnosticFactory0.create(WARNING);
-    DiagnosticFactory0<PsiElement> LOCAL_CLASSES_NOT_SUPPORTED = DiagnosticFactory0.create(ERROR);
+    DiagnosticFactory0<PsiElement> ANONYMOUS_OBJECTS_NOT_SUPPORTED = DiagnosticFactory0.create(ERROR);
     DiagnosticFactory0<PsiElement> INNER_CLASSES_NOT_SUPPORTED = DiagnosticFactory0.create(ERROR);
 
     DiagnosticFactory0<PsiElement> EXPLICIT_SERIALIZABLE_IS_REQUIRED = DiagnosticFactory0.create(WARNING);
@@ -28,6 +28,7 @@ public interface SerializationErrors {
     DiagnosticFactory1<PsiElement, KotlinType> SERIALIZER_NOT_FOUND = DiagnosticFactory1.create(ERROR);
     DiagnosticFactory2<PsiElement, KotlinType, KotlinType> SERIALIZER_NULLABILITY_INCOMPATIBLE = DiagnosticFactory2.create(ERROR);
     DiagnosticFactory3<PsiElement, KotlinType, KotlinType, KotlinType> SERIALIZER_TYPE_INCOMPATIBLE = DiagnosticFactory3.create(WARNING);
+    DiagnosticFactory1<PsiElement, KotlinType> LOCAL_SERIALIZER_USAGE = DiagnosticFactory1.create(ERROR);
     DiagnosticFactory0<PsiElement> TRANSIENT_MISSING_INITIALIZER = DiagnosticFactory0.create(ERROR);
 
     DiagnosticFactory0<PsiElement> TRANSIENT_IS_REDUNDANT = DiagnosticFactory0.create(WARNING);

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginDeclarationChecker.kt

@@ -29,6 +29,7 @@ import org.jetbrains.kotlinx.serialization.compiler.backend.common.bodyPropertie
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.findTypeSerializerOrContextUnchecked
 import org.jetbrains.kotlinx.serialization.compiler.backend.common.primaryConstructorPropertiesDescriptorsMap
 import org.jetbrains.kotlinx.serialization.compiler.resolve.*
+import org.jetbrains.kotlinx.serialization.compiler.resolve.SerializationAnnotations.serializableAnnotationFqName
 
 internal val SERIALIZABLE_PROPERTIES: WritableSlice<ClassDescriptor, SerializableProperties> = Slices.createSimpleSlice()
 
@@ -110,7 +111,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             return false
         }
 
-        if (!descriptor.annotations.hasAnnotation(SerializationAnnotations.serializableAnnotationFqName)) return false
+        if (!descriptor.annotations.hasAnnotation(serializableAnnotationFqName)) return false
 
         if (!serializationPluginEnabledOn(descriptor)) {
             trace.reportOnSerializableAnnotation(descriptor, SerializationErrors.PLUGIN_IS_NOT_ENABLED)
@@ -118,8 +119,8 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         }
 
 
-        if (DescriptorUtils.isLocal(descriptor)) {
-            trace.reportOnSerializableAnnotation(descriptor, SerializationErrors.LOCAL_CLASSES_NOT_SUPPORTED)
+        if (descriptor.isAnonymousObjectOrContained) {
+            trace.reportOnSerializableAnnotation(descriptor, SerializationErrors.ANONYMOUS_OBJECTS_NOT_SUPPORTED)
             return false
         }
 
@@ -140,7 +141,11 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             }
             return false
         }
-        if (!descriptor.hasSerializableAnnotationWithoutArgs) return false
+        if (!descriptor.hasSerializableAnnotationWithoutArgs) {
+            // defined custom serializer
+            checkClassWithCustomSerializer(descriptor, declaration, trace)
+            return false
+        }
 
         if (descriptor.serializableAnnotationIsUseless) {
             trace.reportOnSerializableAnnotation(descriptor, SerializationErrors.SERIALIZABLE_ANNOTATION_IGNORED)
@@ -158,9 +163,27 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         return true
     }
 
+    private fun checkClassWithCustomSerializer(descriptor: ClassDescriptor, declaration: KtDeclaration, trace: BindingTrace) {
+        val annotationPsi = descriptor.findSerializableAnnotationDeclaration()
+        checkCustomSerializerMatch(descriptor.module, descriptor.defaultType, descriptor, annotationPsi, trace, declaration)
+        checkCustomSerializerIsNotLocal(descriptor.module, descriptor, trace, declaration)
+    }
+
+    private val ClassDescriptor.isAnonymousObjectOrContained: Boolean
+        get() {
+            var current: DeclarationDescriptor? = this
+            while (current != null) {
+                if (DescriptorUtils.isAnonymousObject(current)) {
+                    return true
+                }
+                current = current.containingDeclaration
+            }
+            return false
+        }
+
     private fun ClassDescriptor.isSerializableEnumWithMissingSerializer(): Boolean {
         if (kind != ClassKind.ENUM_CLASS) return false
-        if (annotations.hasAnnotation(SerializationAnnotations.serializableAnnotationFqName)) return false
+        if (annotations.hasAnnotation(serializableAnnotationFqName)) return false
         if (annotations.hasAnySerialAnnotation) return true
         return enumEntries().any { (it.annotations.hasAnySerialAnnotation) }
     }
@@ -173,7 +196,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
     }
 
     private fun buildSerializableProperties(descriptor: ClassDescriptor, trace: BindingTrace): SerializableProperties? {
-        if (!descriptor.annotations.hasAnnotation(SerializationAnnotations.serializableAnnotationFqName)) return null
+        if (!descriptor.annotations.hasAnnotation(serializableAnnotationFqName)) return null
         if (!descriptor.isInternalSerializable) return null
         if (descriptor.hasCompanionObjectAsSerializer) return null // customized by user
 
@@ -234,6 +257,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             if (serializer != null) {
                 val element = ktType?.typeElement
                 checkCustomSerializerMatch(it.module, it.type, it.descriptor, element, trace, propertyPsi)
+                checkCustomSerializerIsNotLocal(it.module, it.descriptor, trace, propertyPsi)
                 checkSerializerNullability(it.type, serializer.defaultType, element, trace, propertyPsi)
                 generatorContextForAnalysis.checkTypeArguments(it.module, it.type, element, trace, propertyPsi)
             } else {
@@ -286,6 +310,7 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
         val serializer = findTypeSerializerOrContextUnchecked(module, type)
         if (serializer != null) {
             checkCustomSerializerMatch(module, type, type, element, trace, fallbackElement)
+            checkCustomSerializerIsNotLocal(module, type, trace, fallbackElement)
             checkSerializerNullability(type, serializer.defaultType, element, trace, fallbackElement)
             checkTypeArguments(module, type, element, trace, fallbackElement)
         } else {
@@ -315,6 +340,27 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
             )
     }
 
+    private fun checkCustomSerializerIsNotLocal(
+        module: ModuleDescriptor,
+        declaration: Annotated,
+        trace: BindingTrace,
+        declarationElement: PsiElement
+    ) {
+        val serializerType = declaration.annotations.serializableWith(module) ?: return
+        val serializerDescriptor = serializerType.toClassDescriptor ?: return
+
+        if (DescriptorUtils.isLocal(serializerDescriptor)) {
+            val element = declaration.findSerializableAnnotationDeclaration() ?: declarationElement
+
+            trace.report(
+                SerializationErrors.LOCAL_SERIALIZER_USAGE.on(
+                    element,
+                    serializerType
+                )
+            )
+        }
+    }
+
     private fun checkSerializerNullability(
         classType: KotlinType,
         serializerType: KotlinType,
@@ -344,4 +390,4 @@ open class SerializationPluginDeclarationChecker : DeclarationChecker {
 }
 
 internal val ClassDescriptor.serializableAnnotationIsUseless: Boolean
-    get() = hasSerializableAnnotationWithoutArgs && !isInternalSerializable && !hasCompanionObjectAsSerializer && !isSerializableEnum()
+    get() = hasSerializableAnnotationWithoutArgs && !isInternalSerializable && !hasCompanionObjectAsSerializer && !isSerializableEnum()

plugins/kotlin-serialization/kotlin-serialization-compiler/src/org/jetbrains/kotlinx/serialization/compiler/diagnostic/SerializationPluginErrorsRendering.kt

@@ -26,8 +26,8 @@ object SerializationPluginErrorsRendering : DefaultErrorMessages.Extension {
                     "Make sure that you've setup your buildscript correctly and re-import project."
         )
         MAP.put(
-            SerializationErrors.LOCAL_CLASSES_NOT_SUPPORTED,
-            "Local classes and anonymous objects can not be serializable."
+            SerializationErrors.ANONYMOUS_OBJECTS_NOT_SUPPORTED,
+            "Anonymous objects or contained in it classes can not be serializable."
         )
         MAP.put(
             SerializationErrors.INNER_CLASSES_NOT_SUPPORTED,
@@ -74,6 +74,11 @@ object SerializationPluginErrorsRendering : DefaultErrorMessages.Extension {
             Renderers.RENDER_TYPE,
             Renderers.RENDER_TYPE
         )
+        MAP.put(
+            SerializationErrors.LOCAL_SERIALIZER_USAGE,
+            "Class ''{0}'' can't be used as a serializer since it is local",
+            Renderers.RENDER_TYPE
+        )
         MAP.put(
             SerializationErrors.TRANSIENT_MISSING_INITIALIZER,
             "This property is marked as @Transient and therefore must have an initializing expression"

plugins/kotlin-serialization/kotlin-serialization-compiler/testData/diagnostics/LocalAndAnonymous.kt

@@ -1,17 +1,42 @@
-// !DIAGNOSTICS: -UNUSED_PARAMETER,-UNUSED_VARIABLE
+// !DIAGNOSTICS: -UNUSED_PARAMETER,-UNUSED_VARIABLE,-EXPERIMENTAL_API_USAGE_ERROR,-EXPERIMENTAL_API_USAGE
 // WITH_RUNTIME
 // SKIP_TXT
 
 import kotlinx.serialization.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
 
 fun container() {
-    <!LOCAL_CLASSES_NOT_SUPPORTED!>@Serializable<!>
-    class X
+    @Serializable
+    class X // local classes are allowed
 
-    val y = <!LOCAL_CLASSES_NOT_SUPPORTED!>@Serializable<!> object {}
+    val y = <!ANONYMOUS_OBJECTS_NOT_SUPPORTED!>@Serializable<!> object {
+        fun inObjectFun() {
+            <!ANONYMOUS_OBJECTS_NOT_SUPPORTED!>@Serializable<!>
+            class X // local classes in anonymous object functions are not allowed
+        }
+    }
+
+
+    class LocalSerializer : KSerializer<Any?> {
+        override val descriptor: SerialDescriptor = buildSerialDescriptor("tmp", PrimitiveKind.INT)
+        override fun serialize(encoder: Encoder, value: Any?) {
+            encoder.encodeNull()
+        }
+
+        override fun deserialize(decoder: Decoder): Any? {
+            return decoder.decodeNull()
+        }
+    }
+
+    @Serializable
+    class WithLocalSerializerInProperty(<!LOCAL_SERIALIZER_USAGE!>@Serializable(with = LocalSerializer::class)<!> val x: Any?)
+
+    <!LOCAL_SERIALIZER_USAGE, SERIALIZER_TYPE_INCOMPATIBLE!>@Serializable(with = LocalSerializer::class)<!>
+    data class WithLocalSerializer(val i: Int)
 }
 
-val topLevelAnon = <!LOCAL_CLASSES_NOT_SUPPORTED!>@Serializable<!> object {}
+val topLevelAnon = <!ANONYMOUS_OBJECTS_NOT_SUPPORTED!>@Serializable<!> object {}
 
 @Serializable class A {
     @Serializable class B // nested classes are allowed
@@ -19,4 +44,4 @@ val topLevelAnon = <!LOCAL_CLASSES_NOT_SUPPORTED!>@Serializable<!> object {}
     <!INNER_CLASSES_NOT_SUPPORTED!>@Serializable<!> inner class C // inner classes are not
 
     @Serializable object F {} // regular named object, OK
-}
+}

plugins/kotlin-serialization/kotlin-serialization-compiler/testData/diagnostics/SerializerTypeIncompatible.kt

@@ -14,6 +14,9 @@ object BazSerializer: KSerializer<Baz>
 @Serializer(forClass = Baz::class)
 object NullableBazSerializer: KSerializer<Baz?>
 
+<!SERIALIZER_TYPE_INCOMPATIBLE!>@Serializable(with = BazSerializer::class)<!>
+class Biz(val i: Int)
+
 @Serializable
 class Foo(@Serializable(with = BazSerializer::class) val i: <!SERIALIZER_TYPE_INCOMPATIBLE!>Bar<!>)