[KxSerialization] Fix NPE if sealed class has self-referencing property

Fixes Kotlin/kotlinx.serialization#2294

If the serializable class contains itself as a property, and its serializer is not an object (for example, for a sealed class), in this case there is a race in the initialization of the serializer and child serializers.

To avoid this the serialization class should not cache its own serializer as a child.

Merge-request: KT-MR-10019
Merged-by: Sergey Shanshin <Sergey.Shanshin@jetbrains.com>

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -429,14 +429,24 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
     }
 
     fun IrClass.createCachedChildSerializers(
+        serializableClass: IrClass,
         serializableProperties: List<IrSerializableProperty>
     ): List<IrExpression?> {
         return DeclarationIrBuilder(compilerContext, symbol).run {
-            serializableProperties.map { cacheableChildSerializerInstance(it) }
+            serializableProperties.map { cacheableChildSerializerInstance(serializableClass, it) }
         }
     }
 
-    private fun IrBuilderWithScope.cacheableChildSerializerInstance(property: IrSerializableProperty): IrExpression? {
+    private fun IrBuilderWithScope.cacheableChildSerializerInstance(
+        serializableClass: IrClass,
+        property: IrSerializableProperty
+    ): IrExpression? {
+        // to avoid a cyclical dependency between the serializer cache and the cache of child serializers,
+        // the class  should not cache its serializer as a child
+        if (serializableClass.symbol == property.type.classifier) {
+            return null
+        }
+
         val serializer = getIrSerialTypeInfo(property, compilerContext).serializer ?: return null
         if (serializer.owner.kind == ClassKind.OBJECT) return null
 

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializableIrGenerator.kt

@@ -55,7 +55,7 @@ class SerializableIrGenerator(
     private val IrClass.isInternalSerializable: Boolean get() = kind == ClassKind.CLASS && hasSerializableOrMetaAnnotationWithoutArgs()
 
     private val cachedChildSerializers: List<IrExpression?> =
-        irClass.companionObject()!!.createCachedChildSerializers(properties.serializableProperties)
+        irClass.companionObject()!!.createCachedChildSerializers(irClass, properties.serializableProperties)
 
     private val cachedChildSerializersProperty: IrProperty? =
         irClass.companionObject()!!.addCachedChildSerializersProperty(cachedChildSerializers)

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializerIrGenerator.kt

@@ -89,7 +89,7 @@ open class SerializerIrGenerator(
 
     // non-object serializers which can be cached
     private val cacheableChildSerializers by lazy {
-        serializableIrClass.createCachedChildSerializers(properties.serializableProperties).map { it != null }
+        serializableIrClass.createCachedChildSerializers(serializableIrClass, properties.serializableProperties).map { it != null }
     }
 
     // null if was not found — we're in FIR

plugins/kotlinx-serialization/testData/boxIr/caching.kt

@@ -0,0 +1,29 @@
+// TARGET_BACKEND: JVM_IR
+
+// WITH_STDLIB
+
+import kotlinx.serialization.*
+
+// Serializers for classes containing themselves as properties should be cached correctly
+@Serializable
+class Plain(val p: Plain? = null)
+
+@Serializable
+sealed class Sealed(val p: Sealed? = null)
+
+@Serializable
+open class Open(val p: Open? = null)
+
+@Serializable
+abstract class Abstract(val p: Abstract? = null)
+
+
+fun box(): String {
+    // A correctly cached class must be initialized correctly in order to exclude cyclic nesting of caches
+    Plain.serializer()
+    Sealed.serializer()
+    Open.serializer()
+    Abstract.serializer()
+
+    return "OK"
+}

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationFirLightTreeBlackBoxTestGenerated.java

@@ -33,6 +33,12 @@ public class SerializationFirLightTreeBlackBoxTestGenerated extends AbstractSeri
             runTest("plugins/kotlinx-serialization/testData/boxIr/annotationsOnFile.kt");
         }
 
+        @Test
+        @TestMetadata("caching.kt")
+        public void testCaching() throws Exception {
+            runTest("plugins/kotlinx-serialization/testData/boxIr/caching.kt");
+        }
+
         @Test
         @TestMetadata("classSerializerAsObject.kt")
         public void testClassSerializerAsObject() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationIrBoxTestGenerated.java

@@ -31,6 +31,12 @@ public class SerializationIrBoxTestGenerated extends AbstractSerializationIrBoxT
         runTest("plugins/kotlinx-serialization/testData/boxIr/annotationsOnFile.kt");
     }
 
+    @Test
+    @TestMetadata("caching.kt")
+    public void testCaching() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/boxIr/caching.kt");
+    }
+
     @Test
     @TestMetadata("classSerializerAsObject.kt")
     public void testClassSerializerAsObject() throws Exception {