Make intrinsics more consistent with reflective serializer() lookup

Intrinsic should not look into use-site `@Serializable(with)` annotations.

#KT-64920 Fixed

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializationJvmIrIntrinsicSupport.kt

@@ -165,7 +165,7 @@ class SerializationJvmIrIntrinsicSupport(
         get() = currentVersion != null && currentVersion!! >= ApiVersion.parse("1.2.0")!!
 
     private fun findTypeSerializerOrContext(argType: IrType): IrClassSymbol? =
-        emptyGenerator.findTypeSerializerOrContextUnchecked(this, argType)
+        emptyGenerator.findTypeSerializerOrContextUnchecked(this, argType, useTypeAnnotations = false)
 
     private fun instantiateObject(iv: InstructionAdapter, objectSymbol: IrClassSymbol) {
         val originalIrClass = objectSymbol.owner

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializerSearchUtil.kt

@@ -85,17 +85,17 @@ fun BaseIrGenerator?.findTypeSerializerOrContext(
 }
 
 fun BaseIrGenerator?.findTypeSerializerOrContextUnchecked(
-    context: SerializationBaseContext, kType: IrType
+    context: SerializationBaseContext, kType: IrType, useTypeAnnotations: Boolean = true
 ): IrClassSymbol? {
-    val annotations = kType.annotations
     if (kType.isTypeParameter()) return null
+    val annotations = if (useTypeAnnotations) kType.annotations else emptyList()
     annotations.serializableWith()?.let { return it }
     this?.additionalSerializersInScopeOfCurrentFile?.get(kType.classOrNull!! to kType.isNullable())?.let {
         return it
     }
     if (kType.isMarkedNullable()) return findTypeSerializerOrContextUnchecked(context, kType.makeNotNull())
     if (this?.contextualKClassListInCurrentFile?.contains(kType.classOrNull) == true) return context.referenceClassId(contextSerializerId)
-    return analyzeSpecialSerializers(context, annotations) ?: findTypeSerializer(context, kType)
+    return analyzeSpecialSerializers(context, annotations) ?: findTypeSerializer(context, kType, useTypeAnnotations)
 }
 
 fun analyzeSpecialSerializers(
@@ -111,8 +111,8 @@ fun analyzeSpecialSerializers(
 }
 
 
-fun findTypeSerializer(context: SerializationBaseContext, type: IrType): IrClassSymbol? {
-    type.overriddenSerializer?.let { return it }
+fun findTypeSerializer(context: SerializationBaseContext, type: IrType, useTypeAnnotations: Boolean = true): IrClassSymbol? {
+    if (useTypeAnnotations) type.overriddenSerializer?.let { return it }
     if (type.isTypeParameter()) return null
     if (type.isArray()) return context.referenceClassId(referenceArraySerializerId)
     if (type.isGeneratedSerializableObject()) return context.referenceClassId(objectSerializerId)

plugins/kotlinx-serialization/testData/boxIr/intrinsicsConsistency.kt

@@ -0,0 +1,92 @@
+// TARGET_BACKEND: JVM_IR
+
+// WITH_STDLIB
+
+import kotlinx.serialization.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.encoding.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.json.internal.*
+import kotlinx.serialization.modules.*
+import kotlin.reflect.*
+import kotlin.test.*
+
+class UUID
+
+typealias Uuid = @Serializable(UuidStringSerializer::class) UUID
+
+internal object UuidStringSerializer : KSerializer<UUID> {
+    override val descriptor: SerialDescriptor =
+        PrimitiveSerialDescriptor("UUID", PrimitiveKind.STRING)
+    override fun deserialize(decoder: Decoder): UUID = TODO()
+    override fun serialize(encoder: Encoder, value: UUID): Unit = TODO()
+}
+
+
+@Serializable
+class Box<T>
+
+internal class BoxSerializer<T>(val tSer: KSerializer<T>): KSerializer<Box<T>> {
+    override val descriptor: SerialDescriptor
+        get() = PrimitiveSerialDescriptor("Box@${tSer.descriptor.serialName}", PrimitiveKind.STRING)
+
+    override fun serialize(encoder: Encoder, value: Box<T>) = TODO()
+
+    override fun deserialize(decoder: Decoder): Box<T> = TODO()
+}
+
+typealias BoxAlias<T> = @Serializable(BoxSerializer::class) Box<T>
+
+
+@Serializable
+class ByDefault
+
+internal object NotDefaultSerializer: KSerializer<ByDefault> {
+    override val descriptor: SerialDescriptor =
+        PrimitiveSerialDescriptor("NotByDefault", PrimitiveKind.STRING)
+    override fun deserialize(decoder: Decoder): ByDefault = TODO()
+    override fun serialize(encoder: Encoder, value: ByDefault): Unit = TODO()
+}
+
+typealias NotDefault = @Serializable(NotDefaultSerializer::class) ByDefault
+
+inline fun <reified T> MutableList<String>.isIntrinsicEqual(testName: String) {
+    val r1 = runCatching { serializer<T>() }
+    val r2 = runCatching { serializer(typeOf<T>()) }
+    val areEq = (r1.isFailure && r2.isFailure) || (r1.isSuccess && r2.isSuccess)
+    if (!areEq) {
+        add("Results are not equal for $testName: $r1 vs $r2")
+    } else if (r1.isSuccess) {
+        if (r1.getOrNull()?.descriptor != r2.getOrNull()?.descriptor)
+            add("Not equal descriptors: $r1 vs $r2")
+    }
+}
+
+fun box(): String {
+    val res = buildList {
+        // Check that all variations of @Serializable do not change result of lookup
+        isIntrinsicEqual<ByDefault>("ByDefault")
+        isIntrinsicEqual<@Serializable(NotDefaultSerializer::class) ByDefault>("@Serializable(with) ByDefault")
+        isIntrinsicEqual<NotDefault>("typealias NotDefault")
+        isIntrinsicEqual<List<ByDefault>>("List ByDefault")
+        isIntrinsicEqual<List<@Serializable(NotDefaultSerializer::class) ByDefault>>("List @Serializable(with) ByDefault")
+        isIntrinsicEqual<List<NotDefault>>("List typealias NotDefault")
+
+        // Same, but for generics
+        isIntrinsicEqual<@Serializable(BoxSerializer::class) Box<String>>("@Serializable Box")
+        isIntrinsicEqual<BoxAlias<String>>("typealias Box")
+        isIntrinsicEqual<List<BoxAlias<String>>>("List typealias Box")
+        isIntrinsicEqual<List<@Serializable(BoxSerializer::class) Box<String>>>("List @Serializable Box")
+
+
+        // Same, but for non-serializable class — should be failures
+        isIntrinsicEqual<UUID>("UUID")
+        isIntrinsicEqual<@Serializable(UuidStringSerializer::class) UUID>("@Serializable(with)")
+        isIntrinsicEqual<Uuid>("typealias")
+        isIntrinsicEqual<List<UUID>>("List UUID")
+        isIntrinsicEqual<List<@Serializable(UuidStringSerializer::class) UUID>>("List @Serializable(with)")
+        isIntrinsicEqual<List<Uuid>>("List typealias")
+    }
+    return if (res.isEmpty()) "OK"
+    else res.toString()
+}

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationFirLightTreeBlackBoxTestGenerated.java

@@ -171,6 +171,12 @@ public class SerializationFirLightTreeBlackBoxTestGenerated extends AbstractSeri
       runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsBox.kt");
     }
 
+    @Test
+    @TestMetadata("intrinsicsConsistency.kt")
+    public void testIntrinsicsConsistency() {
+      runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsConsistency.kt");
+    }
+
     @Test
     @TestMetadata("intrinsicsNonReified.kt")
     public void testIntrinsicsNonReified() {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationIrBoxTestGenerated.java

@@ -168,6 +168,12 @@ public class SerializationIrBoxTestGenerated extends AbstractSerializationIrBoxT
     runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsBox.kt");
   }
 
+  @Test
+  @TestMetadata("intrinsicsConsistency.kt")
+  public void testIntrinsicsConsistency() {
+    runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsConsistency.kt");
+  }
+
   @Test
   @TestMetadata("intrinsicsNonReified.kt")
   public void testIntrinsicsNonReified() {