JVM_IR KT-46562 don't use LambdaMetafactory for Serializable SAMs

TODO support serializable lambdas creation with LambdaMetafactory

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java

@@ -20931,6 +20931,18 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
                 runTest("compiler/testData/codegen/box/invokedynamic/sam/samExtFunWithCapturingLambda.kt");
             }
 
+            @Test
+            @TestMetadata("serializableSam1.kt")
+            public void testSerializableSam1() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam1.kt");
+            }
+
+            @Test
+            @TestMetadata("serializableSam2.kt")
+            public void testSerializableSam2() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam2.kt");
+            }
+
             @Test
             @TestMetadata("simpleFunInterfaceConstructor.kt")
             public void testSimpleFunInterfaceConstructor() throws Exception {

compiler/ir/backend.common/src/org/jetbrains/kotlin/ir/util/IrTypeUtils.kt

@@ -143,8 +143,24 @@ private fun collectAllSupertypes(irType: IrSimpleType, result: MutableSet<IrSimp
 // for the class C, this function constructs:
 //      { B<List<Z>>, A<List<List<Z>>, Any }
 // where Z is a type parameter of class C.
-fun getAllSubstitutedSupertypes(irClass: IrClass): MutableSet<IrSimpleType> {
+fun getAllSubstitutedSupertypes(irClass: IrClass): Set<IrSimpleType> {
     val result = HashSet<IrSimpleType>()
     collectAllSupertypes(irClass.defaultType, result)
     return result
 }
+
+private fun collectAllSuperclasses(irClass: IrClass, set: MutableSet<IrClass>) {
+    for (superType in irClass.superTypes) {
+        val classifier = superType.classifierOrNull as? IrClassSymbol ?: continue
+        val superClass = classifier.owner
+        if (set.add(superClass)) {
+            collectAllSuperclasses(superClass, set)
+        }
+    }
+}
+
+fun IrClass.getAllSuperclasses(): Set<IrClass> {
+    val result = HashSet<IrClass>()
+    collectAllSuperclasses(this, result)
+    return result
+}

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/lower/indy/LambdaMetafactoryArguments.kt

@@ -28,6 +28,7 @@ import org.jetbrains.kotlin.ir.types.*
 import org.jetbrains.kotlin.ir.util.*
 import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
 import org.jetbrains.kotlin.load.java.JvmAnnotationNames
+import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.types.Variance
 import org.jetbrains.kotlin.utils.addIfNotNull
@@ -59,6 +60,11 @@ internal class LambdaMetafactoryArgumentsBuilder(
         if (!reference.origin.isLambda && !samClass.isFromJava())
             return null
 
+        // Don't use JDK LambdaMetafactory for serializable lambdas
+        // TODO implement support for serializable lambdas with LambdaMetafactory (requires additional code for deserialization)
+        if (samClass.isInheritedFromSerializable())
+            return null
+
         val samMethod = samClass.getSingleAbstractMethod()
             ?: throw AssertionError("SAM class has no single abstract method: ${samClass.render()}")
 
@@ -122,6 +128,12 @@ internal class LambdaMetafactoryArgumentsBuilder(
         return getLambdaMetafactoryArgsOrNullInner(reference, samMethod, samType, implFun)
     }
 
+    private val javaIoSerializableFqn =
+        FqName("java.io").child(Name.identifier("Serializable"))
+
+    private fun IrClass.isInheritedFromSerializable(): Boolean =
+        getAllSuperclasses().any { it.fqNameWhenAvailable == javaIoSerializableFqn }
+
     private fun IrClass.requiresDelegationToDefaultImpls(): Boolean {
         for (irMemberFun in functions) {
             if (irMemberFun.modality == Modality.ABSTRACT)

compiler/testData/codegen/box/invokedynamic/sam/serializableSam1.kt

@@ -0,0 +1,31 @@
+// TARGET_BACKEND: JVM
+// JVM_TARGET: 1.8
+// SAM_CONVERSIONS: INDY
+// FULL_JDK
+
+// CHECK_BYTECODE_TEXT
+// JVM_IR_TEMPLATES
+// 0 java/lang/invoke/LambdaMetafactory
+
+// FILE: serializableSam1.kt
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.ObjectInputStream
+import java.io.ObjectOutputStream
+
+fun roundtrip(r: Ok): String {
+    val output = ByteArrayOutputStream()
+    ObjectOutputStream(output).writeObject(r)
+    val input = ByteArrayInputStream(output.toByteArray())
+    val rr = ObjectInputStream(input).readObject() as Ok
+    return rr.ok()
+}
+
+fun box() = roundtrip { "OK" }
+
+// FILE: Ok.java
+import java.io.Serializable;
+
+public interface Ok extends Serializable {
+    String ok();
+}

compiler/testData/codegen/box/invokedynamic/sam/serializableSam2.kt

@@ -0,0 +1,37 @@
+// TARGET_BACKEND: JVM
+// JVM_TARGET: 1.8
+// SAM_CONVERSIONS: INDY
+// FULL_JDK
+
+// CHECK_BYTECODE_TEXT
+// JVM_IR_TEMPLATES
+// 0 java/lang/invoke/LambdaMetafactory
+
+// FILE: serializableSam2.kt
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.ObjectInputStream
+import java.io.ObjectOutputStream
+
+fun roundtrip(r: Ok): String {
+    val output = ByteArrayOutputStream()
+    ObjectOutputStream(output).writeObject(r)
+    val input = ByteArrayInputStream(output.toByteArray())
+    val rr = ObjectInputStream(input).readObject() as Ok
+    return rr.ok()
+}
+
+fun box() = roundtrip { "OK" }
+
+// FILE: Ok.java
+public interface Ok extends Base, Marker {}
+
+// FILE: Base.java
+public interface Base {
+    String ok();
+}
+
+// FILE: Marker.java
+import java.io.Serializable;
+
+public interface Marker extends Serializable {}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java

@@ -20907,6 +20907,18 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
                 runTest("compiler/testData/codegen/box/invokedynamic/sam/samExtFunWithCapturingLambda.kt");
             }
 
+            @Test
+            @TestMetadata("serializableSam1.kt")
+            public void testSerializableSam1() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam1.kt");
+            }
+
+            @Test
+            @TestMetadata("serializableSam2.kt")
+            public void testSerializableSam2() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam2.kt");
+            }
+
             @Test
             @TestMetadata("simpleFunInterfaceConstructor.kt")
             public void testSimpleFunInterfaceConstructor() throws Exception {

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java

@@ -20931,6 +20931,18 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
                 runTest("compiler/testData/codegen/box/invokedynamic/sam/samExtFunWithCapturingLambda.kt");
             }
 
+            @Test
+            @TestMetadata("serializableSam1.kt")
+            public void testSerializableSam1() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam1.kt");
+            }
+
+            @Test
+            @TestMetadata("serializableSam2.kt")
+            public void testSerializableSam2() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam2.kt");
+            }
+
             @Test
             @TestMetadata("simpleFunInterfaceConstructor.kt")
             public void testSimpleFunInterfaceConstructor() throws Exception {

compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -17478,6 +17478,16 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
                 runTest("compiler/testData/codegen/box/invokedynamic/sam/samExtFunWithCapturingLambda.kt");
             }
 
+            @TestMetadata("serializableSam1.kt")
+            public void testSerializableSam1() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam1.kt");
+            }
+
+            @TestMetadata("serializableSam2.kt")
+            public void testSerializableSam2() throws Exception {
+                runTest("compiler/testData/codegen/box/invokedynamic/sam/serializableSam2.kt");
+            }
+
             @TestMetadata("simpleFunInterfaceConstructor.kt")
             public void testSimpleFunInterfaceConstructor() throws Exception {
                 runTest("compiler/testData/codegen/box/invokedynamic/sam/simpleFunInterfaceConstructor.kt");