[KxSerialization] Fix NPE if sealed class has self-referencing in generic parameter

Fixes #KT-58918

If the serializable class contains itself as a generic parameter of the property, and its serializer is not an object (for example, for a sealed class), in this case there is a race in the initialization of the serializer and child serializers.

Merge-request: KT-MR-10363
Merged-by: Sergey Shanshin <Sergey.Shanshin@jetbrains.com>

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -446,6 +446,11 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         if (serializableClass.symbol == property.type.classifier) {
             return null
         }
+        // to avoid a cyclical dependency between the serializer cache and the cache of parametrized child serializers,
+        // the class should not cache its serializer as a Generic parameter of a child
+        if (property.type.checkTypeArgumentsHasSelf(serializableClass.symbol)) {
+            return null
+        }
 
         val serializer = getIrSerialTypeInfo(property, compilerContext).serializer ?: return null
         if (serializer.owner.kind == ClassKind.OBJECT) return null
@@ -459,6 +464,17 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         )
     }
 
+    private fun IrSimpleType.checkTypeArgumentsHasSelf(itselfClass: IrClassSymbol): Boolean {
+        arguments.forEach { typeArgument ->
+            if (typeArgument.typeOrNull?.classifierOrNull == itselfClass) return true
+            if (typeArgument is IrSimpleType) {
+                if (typeArgument.checkTypeArgumentsHasSelf(itselfClass)) return true
+            }
+        }
+
+        return false
+    }
+
     fun IrBuilderWithScope.serializerInstance(
         serializerClassOriginal: IrClassSymbol?,
         pluginContext: SerializationPluginContext,

plugins/kotlinx-serialization/testData/boxIr/caching.kt

@@ -17,6 +17,20 @@ open class Open(val p: Open? = null)
 @Serializable
 abstract class Abstract(val p: Abstract? = null)
 
+@Serializable
+class GenericHolder<T> {
+    var value: T? = null
+}
+
+@Serializable
+sealed class SealedGeneric {
+    var holder = GenericHolder<SealedGeneric>()
+}
+
+@Serializable
+sealed class SealedListGeneric {
+    var holder = GenericHolder<GenericHolder<List<SealedListGeneric>>>()
+}
 
 fun box(): String {
     // A correctly cached class must be initialized correctly in order to exclude cyclic nesting of caches
@@ -24,6 +38,8 @@ fun box(): String {
     Sealed.serializer()
     Open.serializer()
     Abstract.serializer()
+    SealedGeneric.serializer()
+    SealedListGeneric.serializer()
 
     return "OK"
 }