[KxSerialization] Fix NPE if sealed class has self-referencing in generic parameter

Fixes #KT-58918

If the serializable class contains itself as a generic parameter of the property, and its serializer is not an object (for example, for a sealed class), in this case there is a race in the initialization of the serializer and child serializers.

Merge-request: KT-MR-10363
Merged-by: Sergey Shanshin <Sergey.Shanshin@jetbrains.com>

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -446,6 +446,11 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         if (serializableClass.symbol == property.type.classifier) {
             return null
         }
+        // to avoid a cyclical dependency between the serializer cache and the cache of parametrized child serializers,
+        // the class should not cache its serializer as a Generic parameter of a child
+        if (property.type.checkTypeArgumentsHasSelf(serializableClass.symbol)) {
+            return null
+        }
 
         val serializer = getIrSerialTypeInfo(property, compilerContext).serializer ?: return null
         if (serializer.owner.kind == ClassKind.OBJECT) return null
@@ -459,6 +464,17 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         )
     }
 
+    private fun IrSimpleType.checkTypeArgumentsHasSelf(itselfClass: IrClassSymbol): Boolean {
+        arguments.forEach { typeArgument ->
+            if (typeArgument.typeOrNull?.classifierOrNull == itselfClass) return true
+            if (typeArgument is IrSimpleType) {
+                if (typeArgument.checkTypeArgumentsHasSelf(itselfClass)) return true
+            }
+        }
+
+        return false
+    }
+
     fun IrBuilderWithScope.serializerInstance(
         serializerClassOriginal: IrClassSymbol?,
         pluginContext: SerializationPluginContext,