Fix generation of checkcast instructions for inline classes

Normalize LHS and RHS types to use only boxed ones and then let bytecode
 optmizer reduce redundant boxing

compiler/backend/src/org/jetbrains/kotlin/codegen/ExpressionCodegen.java

@@ -4431,9 +4431,17 @@ The "returned" value of try expression with no finally is either the last expres
 
         StackValue value = genQualified(receiver, left);
 
+        KotlinType leftKotlinType = value.kotlinType;
+        Type boxedLeftType;
+        if (leftKotlinType != null && InlineClassesUtilsKt.isInlineClassType(leftKotlinType)) {
+            boxedLeftType = typeMapper.mapTypeAsDeclaration(leftKotlinType);
+        } else {
+            boxedLeftType = boxType(value.type);
+        }
+
         Type boxedRightType = boxType(typeMapper.mapTypeAsDeclaration(rightKotlinType));
         return StackValue.operation(boxedRightType, rightKotlinType, v -> {
-            value.put(boxType(value.type), value.kotlinType, v);
+            value.put(boxedLeftType, value.kotlinType, v);
 
             if (value.type == Type.VOID_TYPE) {
                 StackValue.putUnitInstance(v);

compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt

@@ -0,0 +1,48 @@
+// !LANGUAGE: +InlineClasses
+
+inline class AsAny<T>(val x: Any?)
+inline class AsInt(val x: Int)
+
+inline fun <reified T> Any?.checkcast(): T = this as T
+
+object Reference {
+    fun <T, R> transform(a: AsAny<T>): AsAny<R> = a as AsAny<R>
+    fun <T, R> transformNullable(a: AsAny<T>?): AsAny<R> = a as AsAny<R>
+    fun <T, R> transformToNullable(a: AsAny<T>): AsAny<R>? = a as AsAny<R>
+    fun <T, R> transformToNullableTarget(a: AsAny<T>): AsAny<R>? = a as AsAny<R>?
+    fun <T, R> transformNullableToNullableTarget(a: AsAny<T>?): AsAny<R>? = a as AsAny<R>?
+}
+
+object Primitive {
+    fun transform(a: AsInt): AsInt = a as AsInt
+    fun transformNullable(a: AsInt?): AsInt = a as AsInt
+    fun transformToNullable(a: AsInt): AsInt? = a as AsInt
+    fun transformToNullableTarget(a: AsInt): AsInt? = a as AsInt?
+    fun transformNullableToNullableTarget(a: AsInt?): AsInt? = a as AsInt?
+}
+
+fun box(): String {
+    val a = AsAny<Int>(42)
+    val b1 = Reference.transform<Int, Number>(a)
+    val b2 = Reference.transformNullable<Int, Number>(a)
+    val b3 = Reference.transformToNullable<Int, Number>(a)
+    val b4 = Reference.transformToNullableTarget<Int, Number>(a)
+    val b5 = Reference.transformNullableToNullableTarget<Int, Number>(a)
+    val b6 = Reference.transformNullableToNullableTarget<Int, Number>(null)
+
+    val b7 = a.checkcast<AsAny<Number>>()
+    if (b7.x != a.x) return "Fail 1"
+
+    val c = AsInt(42)
+    val d1 = Primitive.transform(c)
+    val d2 = Primitive.transformNullable(c)
+    val d3 = Primitive.transformToNullable(c)
+    val d4 = Primitive.transformToNullableTarget(c)
+    val d5 = Primitive.transformNullableToNullableTarget(c)
+    val d6 = Primitive.transformNullableToNullableTarget(null)
+
+    val d7 = c.checkcast<AsInt>()
+    if (d7.x != c.x) return "Fail 2"
+
+    return "OK"
+}

compiler/testData/codegen/bytecodeText/inlineClasses/noBoxingOnCastOperations.kt

@@ -0,0 +1,30 @@
+// !LANGUAGE: +InlineClasses
+
+// FILE: util.kt
+
+inline class AsAny<T>(val x: Any?)
+inline class AsInt(val x: Int)
+
+// FILE: Reference.kt
+
+fun <T, R> transform(a: AsAny<T>): AsAny<R> = a as AsAny<R>
+fun <T, R> transformNullable(a: AsAny<T>?): AsAny<R> = a as AsAny<R> // unbox
+fun <T, R> transformToNullable(a: AsAny<T>): AsAny<R>? = a as AsAny<R> // box
+fun <T, R> transformToNullableTarget(a: AsAny<T>): AsAny<R>? = a as AsAny<R>? // box
+fun <T, R> transformNullableToNullableTarget(a: AsAny<T>?): AsAny<R>? = a as AsAny<R>?
+
+// FILE: Primitive.kt
+
+fun transform(a: AsInt): AsInt = a as AsInt
+fun transformNullable(a: AsInt?): AsInt = a as AsInt // unbox
+fun transformToNullable(a: AsInt): AsInt? = a as AsInt // box
+fun transformToNullableTarget(a: AsInt): AsInt? = a as AsInt? // box
+fun transformNullableToNullableTarget(a: AsInt?): AsInt? = a as AsInt?
+
+// @ReferenceKt.class:
+// 2 INVOKESTATIC AsAny\$Erased.box
+// 1 INVOKEVIRTUAL AsAny.unbox
+
+// @PrimitiveKt.class:
+// 2 INVOKESTATIC AsInt\$Erased.box
+// 1 INVOKEVIRTUAL AsInt.unbox

compiler/tests-ir-jvm/tests/org/jetbrains/kotlin/codegen/ir/IrBlackBoxCodegenTestGenerated.java

@@ -10978,6 +10978,11 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             runTest("compiler/testData/codegen/box/inlineClasses/inlineClassImplementsCollection.kt");
         }
 
+        @TestMetadata("inlineClassesCheckCast.kt")
+        public void testInlineClassesCheckCast() throws Exception {
+            runTest("compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt");
+        }
+
         @TestMetadata("inlineFunctionInsideInlineClass.kt")
         public void testInlineFunctionInsideInlineClass() throws Exception {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineFunctionInsideInlineClass.kt");

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -10978,6 +10978,11 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineClassImplementsCollection.kt");
         }
 
+        @TestMetadata("inlineClassesCheckCast.kt")
+        public void testInlineClassesCheckCast() throws Exception {
+            runTest("compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt");
+        }
+
         @TestMetadata("inlineFunctionInsideInlineClass.kt")
         public void testInlineFunctionInsideInlineClass() throws Exception {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineFunctionInsideInlineClass.kt");

compiler/tests/org/jetbrains/kotlin/codegen/BytecodeTextTestGenerated.java

@@ -2060,6 +2060,11 @@ public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
             runTest("compiler/testData/codegen/bytecodeText/inlineClasses/noAssertionsForInlineClassesBasedOnNullableTypes.kt");
         }
 
+        @TestMetadata("noBoxingOnCastOperations.kt")
+        public void testNoBoxingOnCastOperations() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/inlineClasses/noBoxingOnCastOperations.kt");
+        }
+
         @TestMetadata("noBoxingOperationsOnNonTrivialSpread.kt")
         public void testNoBoxingOperationsOnNonTrivialSpread() throws Exception {
             runTest("compiler/testData/codegen/bytecodeText/inlineClasses/noBoxingOperationsOnNonTrivialSpread.kt");

compiler/tests/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -10978,6 +10978,11 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
             runTest("compiler/testData/codegen/box/inlineClasses/inlineClassImplementsCollection.kt");
         }
 
+        @TestMetadata("inlineClassesCheckCast.kt")
+        public void testInlineClassesCheckCast() throws Exception {
+            runTest("compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt");
+        }
+
         @TestMetadata("inlineFunctionInsideInlineClass.kt")
         public void testInlineFunctionInsideInlineClass() throws Exception {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineFunctionInsideInlineClass.kt");

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/IrJsCodegenBoxTestGenerated.java

@@ -9648,6 +9648,11 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineClassAsLastExpressionInInLambda.kt");
         }
 
+        @TestMetadata("inlineClassesCheckCast.kt")
+        public void testInlineClassesCheckCast() throws Exception {
+            runTest("compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt");
+        }
+
         @TestMetadata("inlineFunctionInsideInlineClass.kt")
         public void testInlineFunctionInsideInlineClass() throws Exception {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineFunctionInsideInlineClass.kt");

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/JsCodegenBoxTestGenerated.java

@@ -9648,6 +9648,11 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineClassAsLastExpressionInInLambda.kt");
         }
 
+        @TestMetadata("inlineClassesCheckCast.kt")
+        public void testInlineClassesCheckCast() throws Exception {
+            runTest("compiler/testData/codegen/box/inlineClasses/inlineClassesCheckCast.kt");
+        }
+
         @TestMetadata("inlineFunctionInsideInlineClass.kt")
         public void testInlineFunctionInsideInlineClass() throws Exception {
             runTest("compiler/testData/codegen/box/inlineClasses/inlineFunctionInsideInlineClass.kt");