[KxSerialization] Fix NPE if sealed class has self-referencing property

Fixes Kotlin/kotlinx.serialization#2294

If the serializable class contains itself as a property, and its serializer is not an object (for example, for a sealed class), in this case there is a race in the initialization of the serializer and child serializers.

To avoid this the serialization class should not cache its own serializer as a child.

Merge-request: KT-MR-10019
Merged-by: Sergey Shanshin <Sergey.Shanshin@jetbrains.com>

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -429,14 +429,24 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
     }
 
     fun IrClass.createCachedChildSerializers(
+        serializableClass: IrClass,
         serializableProperties: List<IrSerializableProperty>
     ): List<IrExpression?> {
         return DeclarationIrBuilder(compilerContext, symbol).run {
-            serializableProperties.map { cacheableChildSerializerInstance(it) }
+            serializableProperties.map { cacheableChildSerializerInstance(serializableClass, it) }
         }
     }
 
-    private fun IrBuilderWithScope.cacheableChildSerializerInstance(property: IrSerializableProperty): IrExpression? {
+    private fun IrBuilderWithScope.cacheableChildSerializerInstance(
+        serializableClass: IrClass,
+        property: IrSerializableProperty
+    ): IrExpression? {
+        // to avoid a cyclical dependency between the serializer cache and the cache of child serializers,
+        // the class  should not cache its serializer as a child
+        if (serializableClass.symbol == property.type.classifier) {
+            return null
+        }
+
         val serializer = getIrSerialTypeInfo(property, compilerContext).serializer ?: return null
         if (serializer.owner.kind == ClassKind.OBJECT) return null
 

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializableIrGenerator.kt

@@ -55,7 +55,7 @@ class SerializableIrGenerator(
     private val IrClass.isInternalSerializable: Boolean get() = kind == ClassKind.CLASS && hasSerializableOrMetaAnnotationWithoutArgs()
 
     private val cachedChildSerializers: List<IrExpression?> =
-        irClass.companionObject()!!.createCachedChildSerializers(properties.serializableProperties)
+        irClass.companionObject()!!.createCachedChildSerializers(irClass, properties.serializableProperties)
 
     private val cachedChildSerializersProperty: IrProperty? =
         irClass.companionObject()!!.addCachedChildSerializersProperty(cachedChildSerializers)

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializerIrGenerator.kt

@@ -89,7 +89,7 @@ open class SerializerIrGenerator(
 
     // non-object serializers which can be cached
     private val cacheableChildSerializers by lazy {
-        serializableIrClass.createCachedChildSerializers(properties.serializableProperties).map { it != null }
+        serializableIrClass.createCachedChildSerializers(serializableIrClass, properties.serializableProperties).map { it != null }
     }
 
     // null if was not found — we're in FIR