Handle non-reified type parameters of function inside serializer<T>() intrinsic

to match an error message thrown from KType-based serializer<T>().

Fixes https://github.com/Kotlin/kotlinx.serialization/issues/2528

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializationJvmIrIntrinsicSupport.kt

@@ -29,7 +29,6 @@ import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.load.kotlin.TypeMappingMode
 import org.jetbrains.kotlin.name.ClassId
 import org.jetbrains.kotlin.resolve.jvm.AsmTypes
-import org.jetbrains.kotlin.types.model.KotlinTypeMarker
 import org.jetbrains.kotlinx.serialization.compiler.backend.jvm.*
 import org.jetbrains.kotlinx.serialization.compiler.backend.jvm.annotationArrayType
 import org.jetbrains.kotlinx.serialization.compiler.backend.jvm.doubleAnnotationArrayType
@@ -250,16 +249,34 @@ class SerializationJvmIrIntrinsicSupport(
             return false
         }
 
-    private fun generateThrowOnStarProjection(parentType: IrSimpleType, adapter: InstructionAdapter) {
-        val iaeName = "java/lang/IllegalArgumentException"
-        with(adapter) {
-            anew(Type.getObjectType(iaeName))
-            dup()
-            aconst("Star projections in type arguments are not allowed, but had ${parentType.render()}")
-            invokespecial(iaeName, "<init>", "(Ljava/lang/String;)V", false)
-            checkcast(Type.getObjectType("java/lang/Throwable"))
-            athrow()
+    private val iaeName = "java/lang/IllegalArgumentException"
+
+    private fun InstructionAdapter.throwIae(message: String) {
+        anew(Type.getObjectType(iaeName))
+        dup()
+        aconst(message)
+        invokespecial(iaeName, "<init>", "(Ljava/lang/String;)V", false)
+        checkcast(Type.getObjectType("java/lang/Throwable"))
+        athrow()
+    }
+
+    private fun IrTypeArgument.argumentTypeOrGenerateException(ownerType: IrSimpleType, adapter: InstructionAdapter): IrType? {
+        val argType = this.typeOrNull
+        if (argType == null) {
+            adapter.throwIae("Star projections in type arguments are not allowed, but had ${ownerType.render()}")
+            return null
         }
+        val classifier = argType.classifierOrNull
+        if (classifier is IrTypeParameterSymbol && !classifier.owner.isReified) {
+            val fullName = argType.render() // T of <function fqName>
+            val name = classifier.owner.name.asString()
+            val message = "Captured type parameter $fullName from generic non-reified function. " +
+                    "Such functionality cannot be supported because $name is erased, either specify serializer explicitly or make " +
+                    "calling function inline with reified $name."
+            adapter.throwIae(message)
+            return null
+        }
+        return argType
     }
 
     fun generateSerializerForType(
@@ -278,10 +295,7 @@ class SerializationJvmIrIntrinsicSupport(
             val companionType = if (typeDescriptor.isSerializableObject) typeDescriptor else typeDescriptor.companionObject()!!
             support.instantiateObject(adapter, companionType.symbol)
             val args = (type as IrSimpleType).arguments.map {
-                it.typeOrNull ?: run {
-                    generateThrowOnStarProjection(type, adapter)
-                    return
-                }
+                it.argumentTypeOrGenerateException(type, adapter) ?: return
             }
             args.forEach { generateSerializerForType(it, adapter, intrinsicType) }
             val signature = kSerializerType.descriptor.repeat(args.size)
@@ -337,10 +351,7 @@ class SerializationJvmIrIntrinsicSupport(
         }
         // serializer is not singleton object and shall be instantiated
         val typeArgumentsAsTypes = (kType as IrSimpleType).arguments.map {
-            it.typeOrNull ?: run {
-                generateThrowOnStarProjection(kType, iv)
-                return false
-            }
+            it.argumentTypeOrGenerateException(kType, iv) ?: return false
         }
         val argSerializers = typeArgumentsAsTypes.map { argType ->
             // check if any type argument is not serializable

plugins/kotlinx-serialization/testData/boxIr/intrinsicsNonReified.kt

@@ -0,0 +1,43 @@
+// TARGET_BACKEND: JVM_IR
+
+// WITH_STDLIB
+
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.internal.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.modules.*
+import java.lang.AssertionError
+import java.lang.IllegalArgumentException
+
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.modules.*
+import kotlin.reflect.typeOf
+
+@Serializable
+class Box<T>(val t: T)
+
+inline fun <reified T> inner() = serializer<T>()
+
+fun <T> outer() = inner<Box<T>>()
+
+fun checkBlock(name: String, block:() -> Unit) {
+    try {
+        block()
+    } catch (e: IllegalArgumentException) {
+        if (!e.message!!.contains("Captured type parameter T of <root>.IntrinsicsNonReifiedKt.outer from generic non-reified function.")) throw e
+        return
+    }
+    throw AssertionError("Expected exception to be thrown in block $name")
+}
+
+fun box(): String {
+    checkBlock("string") {
+        outer<String>()
+    }
+    checkBlock("list string") {
+        outer<List<String>>()
+    }
+    return "OK"
+}

plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.asm.fir.txt

@@ -0,0 +1,86 @@
+public final class Box$$serializer : java/lang/Object, kotlinx/serialization/internal/GeneratedSerializer {
+    private final kotlinx.serialization.descriptors.SerialDescriptor descriptor
+
+    private final kotlinx.serialization.KSerializer typeSerial0
+
+    private void <init>()
+
+    public void <init>(kotlinx.serialization.KSerializer typeSerial0)
+
+    public final kotlinx.serialization.KSerializer[] childSerializers()
+
+    public final Box deserialize(kotlinx.serialization.encoding.Decoder decoder)
+
+    public java.lang.Object deserialize(kotlinx.serialization.encoding.Decoder decoder)
+
+    public final kotlinx.serialization.descriptors.SerialDescriptor getDescriptor()
+
+    private final kotlinx.serialization.KSerializer getTypeSerial0()
+
+    public final void serialize(kotlinx.serialization.encoding.Encoder encoder, Box value)
+
+    public void serialize(kotlinx.serialization.encoding.Encoder encoder, java.lang.Object value)
+
+    public final kotlinx.serialization.KSerializer[] typeParametersSerializers()
+}
+
+public final class Box$Companion : java/lang/Object {
+    private void <init>()
+
+    public void <init>(kotlin.jvm.internal.DefaultConstructorMarker $constructor_marker)
+
+    public final kotlinx.serialization.KSerializer serializer(kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class Box : java/lang/Object {
+    private final static kotlinx.serialization.descriptors.SerialDescriptor $cachedDescriptor
+
+    public final static Box$Companion Companion
+
+    private final java.lang.Object t
+
+    static void <clinit>()
+
+    public void <init>(java.lang.Object t)
+
+    public void <init>(int seen0, java.lang.Object t, kotlinx.serialization.internal.SerializationConstructorMarker serializationConstructorMarker)
+
+    public final java.lang.Object getT()
+
+    public final static void write$Self$main(Box self, kotlinx.serialization.encoding.CompositeEncoder output, kotlinx.serialization.descriptors.SerialDescriptor serialDesc, kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class IntrinsicsNonReifiedKt : java/lang/Object {
+    public final static kotlinx.serialization.KSerializer inner() {
+          ICONST_0
+          ISTORE (0)
+        LABEL (L0)
+        LINENUMBER (12)
+          BIPUSH (6)
+          LDC (T)
+          INVOKESTATIC (kotlin/jvm/internal/Intrinsics, reifiedOperationMarker, (ILjava/lang/String;)V)
+          ACONST_NULL
+          LDC (kotlinx.serialization.serializer.simple)
+          INVOKESTATIC (kotlin/jvm/internal/MagicApiIntrinsics, voidMagicApiCall, (Ljava/lang/Object;)V)
+          INVOKESTATIC (kotlinx/serialization/SerializersKt, serializer, (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;)
+          ARETURN
+        LABEL (L1)
+    }
+
+    public final static kotlinx.serialization.KSerializer outer() {
+        LABEL (L0)
+        LINENUMBER (14)
+          ICONST_0
+          ISTORE (0)
+        LABEL (L1)
+        LINENUMBER (16)
+          GETSTATIC (Box, Companion, LBox$Companion;)
+          NEW (java/lang/IllegalArgumentException)
+          DUP
+          LDC (Captured type parameter T of <root>.IntrinsicsNonReifiedKt.outer from generic non-reified function. Such functionality cannot be supported because T is erased, either specify serializer explicitly or make calling function inline with reified T.)
+          INVOKESPECIAL (java/lang/IllegalArgumentException, <init>, (Ljava/lang/String;)V)
+          CHECKCAST (java/lang/Throwable)
+          ATHROW
+        LABEL (L2)
+    }
+}

plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.asm.ir.txt

@@ -0,0 +1,86 @@
+public final class Box$$serializer : java/lang/Object, kotlinx/serialization/internal/GeneratedSerializer {
+    private final kotlinx.serialization.internal.PluginGeneratedSerialDescriptor descriptor
+
+    private final kotlinx.serialization.KSerializer typeSerial0
+
+    private void <init>()
+
+    public void <init>(kotlinx.serialization.KSerializer typeSerial0)
+
+    public kotlinx.serialization.KSerializer[] childSerializers()
+
+    public Box deserialize(kotlinx.serialization.encoding.Decoder decoder)
+
+    public java.lang.Object deserialize(kotlinx.serialization.encoding.Decoder decoder)
+
+    public kotlinx.serialization.descriptors.SerialDescriptor getDescriptor()
+
+    private final kotlinx.serialization.KSerializer getTypeSerial0()
+
+    public void serialize(kotlinx.serialization.encoding.Encoder encoder, Box value)
+
+    public void serialize(kotlinx.serialization.encoding.Encoder encoder, java.lang.Object value)
+
+    public kotlinx.serialization.KSerializer[] typeParametersSerializers()
+}
+
+public final class Box$Companion : java/lang/Object {
+    private void <init>()
+
+    public void <init>(kotlin.jvm.internal.DefaultConstructorMarker $constructor_marker)
+
+    public final kotlinx.serialization.KSerializer serializer(kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class Box : java/lang/Object {
+    private final static kotlinx.serialization.descriptors.SerialDescriptor $cachedDescriptor
+
+    public final static Box$Companion Companion
+
+    private final java.lang.Object t
+
+    static void <clinit>()
+
+    public void <init>(java.lang.Object t)
+
+    public void <init>(int seen1, java.lang.Object t, kotlinx.serialization.internal.SerializationConstructorMarker serializationConstructorMarker)
+
+    public final java.lang.Object getT()
+
+    public final static void write$Self$main(Box self, kotlinx.serialization.encoding.CompositeEncoder output, kotlinx.serialization.descriptors.SerialDescriptor serialDesc, kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class IntrinsicsNonReifiedKt : java/lang/Object {
+    public final static kotlinx.serialization.KSerializer inner() {
+          ICONST_0
+          ISTORE (0)
+        LABEL (L0)
+        LINENUMBER (12)
+          BIPUSH (6)
+          LDC (T)
+          INVOKESTATIC (kotlin/jvm/internal/Intrinsics, reifiedOperationMarker, (ILjava/lang/String;)V)
+          ACONST_NULL
+          LDC (kotlinx.serialization.serializer.simple)
+          INVOKESTATIC (kotlin/jvm/internal/MagicApiIntrinsics, voidMagicApiCall, (Ljava/lang/Object;)V)
+          INVOKESTATIC (kotlinx/serialization/SerializersKt, serializer, (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;)
+          ARETURN
+        LABEL (L1)
+    }
+
+    public final static kotlinx.serialization.KSerializer outer() {
+        LABEL (L0)
+        LINENUMBER (14)
+          ICONST_0
+          ISTORE (0)
+        LABEL (L1)
+        LINENUMBER (16)
+          GETSTATIC (Box, Companion, LBox$Companion;)
+          NEW (java/lang/IllegalArgumentException)
+          DUP
+          LDC (Captured type parameter T of <root>.IntrinsicsNonReifiedKt.outer from generic non-reified function. Such functionality cannot be supported because T is erased, either specify serializer explicitly or make calling function inline with reified T.)
+          INVOKESPECIAL (java/lang/IllegalArgumentException, <init>, (Ljava/lang/String;)V)
+          CHECKCAST (java/lang/Throwable)
+          ATHROW
+        LABEL (L2)
+    }
+}

plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.asm.txt

@@ -0,0 +1,141 @@
+public final class Box$$serializer : java/lang/Object, kotlinx/serialization/internal/GeneratedSerializer {
+    private final kotlinx.serialization.descriptors.SerialDescriptor $$serialDesc
+
+    private kotlinx.serialization.KSerializer typeSerial0
+
+    private void <init>()
+
+    public void <init>(kotlinx.serialization.KSerializer typeSerial0)
+
+    public kotlinx.serialization.KSerializer[] childSerializers()
+
+    public Box deserialize(kotlinx.serialization.encoding.Decoder decoder)
+
+    public java.lang.Object deserialize(kotlinx.serialization.encoding.Decoder p0)
+
+    public kotlinx.serialization.descriptors.SerialDescriptor getDescriptor()
+
+    public void serialize(kotlinx.serialization.encoding.Encoder encoder, Box value)
+
+    public void serialize(kotlinx.serialization.encoding.Encoder p0, java.lang.Object p1)
+
+    public kotlinx.serialization.KSerializer[] typeParametersSerializers()
+}
+
+public final class Box$Companion : java/lang/Object {
+    private void <init>()
+
+    public void <init>(kotlin.jvm.internal.DefaultConstructorMarker $constructor_marker)
+
+    public final kotlinx.serialization.KSerializer serializer(kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class Box : java/lang/Object {
+    private final static kotlinx.serialization.descriptors.SerialDescriptor $cachedDescriptor
+
+    public final static Box$Companion Companion
+
+    private final java.lang.Object t
+
+    static void <clinit>()
+
+    public void <init>(java.lang.Object t)
+
+    public void <init>(int seen1, java.lang.Object t, kotlinx.serialization.internal.SerializationConstructorMarker serializationConstructorMarker)
+
+    public final java.lang.Object getT()
+
+    public final static void write$Self$main(Box self, kotlinx.serialization.encoding.CompositeEncoder output, kotlinx.serialization.descriptors.SerialDescriptor serialDesc, kotlinx.serialization.KSerializer typeSerial0)
+}
+
+public final class IntrinsicsNonReifiedKt : java/lang/Object {
+    public final static kotlinx.serialization.KSerializer inner() {
+          LDC (0)
+          ISTORE (0)
+        LABEL (L0)
+        LINENUMBER (12)
+          ICONST_0
+          ISTORE (1)
+        LABEL (L1)
+          BIPUSH (6)
+          LDC (T)
+          INVOKESTATIC (kotlin/jvm/internal/Intrinsics, reifiedOperationMarker, (ILjava/lang/String;)V)
+          ACONST_NULL
+        LABEL (L2)
+        LINENUMBER (16)
+          INVOKESTATIC (kotlinx/serialization/SerializersKt, serializer, (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;)
+          ASTORE (2)
+        LABEL (L3)
+          ICONST_0
+          ISTORE (3)
+        LABEL (L4)
+        LINENUMBER (17)
+          ALOAD (2)
+          LDC (null cannot be cast to non-null type kotlinx.serialization.KSerializer<T of kotlinx.serialization.internal.Platform_commonKt.cast>)
+          INVOKESTATIC (kotlin/jvm/internal/Intrinsics, checkNotNull, (Ljava/lang/Object;Ljava/lang/String;)V)
+          ALOAD (2)
+          CHECKCAST (kotlinx/serialization/KSerializer)
+        LABEL (L5)
+        LINENUMBER (16)
+          NOP
+        LABEL (L6)
+        LINENUMBER (12)
+          ARETURN
+        LABEL (L7)
+    }
+
+    public final static kotlinx.serialization.KSerializer outer() {
+        LABEL (L0)
+        LINENUMBER (14)
+          ICONST_0
+          ISTORE (0)
+        LABEL (L1)
+        LINENUMBER (18)
+          ICONST_0
+          ISTORE (1)
+        LABEL (L2)
+          LDC (LBox;)
+          GETSTATIC (kotlin/reflect/KTypeProjection, Companion, Lkotlin/reflect/KTypeProjection$Companion;)
+          NEW (kotlin/jvm/internal/FunctionReferenceImpl)
+          DUP
+          ICONST_0
+          LDC (LIntrinsicsNonReifiedKt;)
+          LDC (outer)
+          LDC (outer()Lkotlinx/serialization/KSerializer;)
+          ICONST_1
+          INVOKESPECIAL (kotlin/jvm/internal/FunctionReferenceImpl, <init>, (ILjava/lang/Class;Ljava/lang/String;Ljava/lang/String;I)V)
+          LDC (T)
+          GETSTATIC (kotlin/reflect/KVariance, INVARIANT, Lkotlin/reflect/KVariance;)
+          ICONST_0
+          INVOKESTATIC (kotlin/jvm/internal/Reflection, typeParameter, (Ljava/lang/Object;Ljava/lang/String;Lkotlin/reflect/KVariance;Z)Lkotlin/reflect/KTypeParameter;)
+          DUP
+          LDC (Ljava/lang/Object;)
+          INVOKESTATIC (kotlin/jvm/internal/Reflection, nullableTypeOf, (Ljava/lang/Class;)Lkotlin/reflect/KType;)
+          INVOKESTATIC (kotlin/jvm/internal/Reflection, setUpperBounds, (Lkotlin/reflect/KTypeParameter;Lkotlin/reflect/KType;)V)
+          INVOKESTATIC (kotlin/jvm/internal/Reflection, typeOf, (Lkotlin/reflect/KClassifier;)Lkotlin/reflect/KType;)
+          INVOKEVIRTUAL (kotlin/reflect/KTypeProjection$Companion, invariant, (Lkotlin/reflect/KType;)Lkotlin/reflect/KTypeProjection;)
+          INVOKESTATIC (kotlin/jvm/internal/Reflection, typeOf, (Ljava/lang/Class;Lkotlin/reflect/KTypeProjection;)Lkotlin/reflect/KType;)
+        LABEL (L3)
+        LINENUMBER (19)
+          INVOKESTATIC (kotlinx/serialization/SerializersKt, serializer, (Lkotlin/reflect/KType;)Lkotlinx/serialization/KSerializer;)
+          ASTORE (2)
+        LABEL (L4)
+          ICONST_0
+          ISTORE (3)
+        LABEL (L5)
+        LINENUMBER (20)
+          ALOAD (2)
+          LDC (null cannot be cast to non-null type kotlinx.serialization.KSerializer<T of kotlinx.serialization.internal.Platform_commonKt.cast>)
+          INVOKESTATIC (kotlin/jvm/internal/Intrinsics, checkNotNull, (Ljava/lang/Object;Ljava/lang/String;)V)
+          ALOAD (2)
+        LABEL (L6)
+        LINENUMBER (19)
+          NOP
+        LABEL (L7)
+        LINENUMBER (18)
+          NOP
+        LABEL (L8)
+        LINENUMBER (14)
+          ARETURN
+    }
+}

plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.kt

@@ -0,0 +1,14 @@
+// CURIOUS_ABOUT: inner, outer
+// WITH_STDLIB
+
+import kotlinx.serialization.*
+import kotlinx.serialization.json.*
+import kotlinx.serialization.modules.*
+import kotlin.reflect.typeOf
+
+@Serializable
+class Box<T>(val t: T)
+
+inline fun <reified T> inner() = serializer<T>()
+
+fun <T> outer() = inner<Box<T>>()

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationAsmLikeInstructionsListingTestGenerated.java

@@ -43,6 +43,12 @@ public class SerializationAsmLikeInstructionsListingTestGenerated extends Abstra
         runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsAdvanced.kt");
     }
 
+    @Test
+    @TestMetadata("IntrinsicsNonReified.kt")
+    public void testIntrinsicsNonReified() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.kt");
+    }
+
     @Test
     @TestMetadata("Sealed.kt")
     public void testSealed() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationFirLightTreeAsmLikeInstructionsListingTestGenerated.java

@@ -43,6 +43,12 @@ public class SerializationFirLightTreeAsmLikeInstructionsListingTestGenerated ex
         runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsAdvanced.kt");
     }
 
+    @Test
+    @TestMetadata("IntrinsicsNonReified.kt")
+    public void testIntrinsicsNonReified() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.kt");
+    }
+
     @Test
     @TestMetadata("Sealed.kt")
     public void testSealed() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationFirLightTreeBlackBoxTestGenerated.java

@@ -165,6 +165,12 @@ public class SerializationFirLightTreeBlackBoxTestGenerated extends AbstractSeri
             runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsBox.kt");
         }
 
+        @Test
+        @TestMetadata("intrinsicsNonReified.kt")
+        public void testIntrinsicsNonReified() throws Exception {
+            runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsNonReified.kt");
+        }
+
         @Test
         @TestMetadata("intrinsicsNullable.kt")
         public void testIntrinsicsNullable() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationIrAsmLikeInstructionsListingTestGenerated.java

@@ -43,6 +43,12 @@ public class SerializationIrAsmLikeInstructionsListingTestGenerated extends Abst
         runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsAdvanced.kt");
     }
 
+    @Test
+    @TestMetadata("IntrinsicsNonReified.kt")
+    public void testIntrinsicsNonReified() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/codegen/IntrinsicsNonReified.kt");
+    }
+
     @Test
     @TestMetadata("Sealed.kt")
     public void testSealed() throws Exception {

plugins/kotlinx-serialization/tests-gen/org/jetbrains/kotlinx/serialization/runners/SerializationIrBoxTestGenerated.java

@@ -163,6 +163,12 @@ public class SerializationIrBoxTestGenerated extends AbstractSerializationIrBoxT
         runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsBox.kt");
     }
 
+    @Test
+    @TestMetadata("intrinsicsNonReified.kt")
+    public void testIntrinsicsNonReified() throws Exception {
+        runTest("plugins/kotlinx-serialization/testData/boxIr/intrinsicsNonReified.kt");
+    }
+
     @Test
     @TestMetadata("intrinsicsNullable.kt")
     public void testIntrinsicsNullable() throws Exception {