Optimize out trivial INSTANCEOF checks

#KT-18157 Fixed Target versions 1.1.4

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/nullCheck/RedundantNullCheckMethodTransformer.kt

@@ -20,12 +20,12 @@ import org.jetbrains.kotlin.codegen.coroutines.withInstructionAdapter
 import org.jetbrains.kotlin.codegen.inline.ReifiedTypeInliner
 import org.jetbrains.kotlin.codegen.optimization.DeadCodeEliminationMethodTransformer
 import org.jetbrains.kotlin.codegen.optimization.common.OptimizationBasicInterpreter
+import org.jetbrains.kotlin.codegen.optimization.common.StrictBasicValue
 import org.jetbrains.kotlin.codegen.optimization.common.isInsn
 import org.jetbrains.kotlin.codegen.optimization.fixStack.peek
 import org.jetbrains.kotlin.codegen.optimization.fixStack.top
 import org.jetbrains.kotlin.codegen.optimization.transformer.MethodTransformer
 import org.jetbrains.kotlin.utils.SmartList
-import org.jetbrains.kotlin.utils.addToStdlib.assertedCast
 import org.jetbrains.org.objectweb.asm.Label
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.jetbrains.org.objectweb.asm.Type
@@ -87,28 +87,30 @@ class RedundantNullCheckMethodTransformer : MethodTransformer() {
         private fun injectNullabilityAssumptions(checkedReferenceTypes: Map<AbstractInsnNode, Type>) =
                 NullabilityAssumptionsBuilder(checkedReferenceTypes).injectNullabilityAssumptions()
 
-        private fun analyzeNullabilities(): Map<AbstractInsnNode, Nullability> {
+        private fun analyzeNullabilities(): Map<AbstractInsnNode, StrictBasicValue> {
             val frames = analyze(internalClassName, methodNode, NullabilityInterpreter())
             val insns = methodNode.instructions.toArray()
-            val nullabilityMap = HashMap<AbstractInsnNode, Nullability>()
+            val nullabilityMap = LinkedHashMap<AbstractInsnNode, StrictBasicValue>()
             for (i in insns.indices) {
-                val nullability = frames[i]?.top()?.getNullability() ?: continue
+                val top = frames[i]?.top() as? StrictBasicValue ?: continue
+                val nullability = top.getNullability()
                 if (nullability == Nullability.NULLABLE) continue
 
                 val insn = insns[i]
                 if (insn.isInstanceOfOrNullCheck()) {
-                    nullabilityMap[insn] = nullability
+                    nullabilityMap[insn] = top
                 }
             }
             return nullabilityMap
         }
 
-        private fun transformTrivialChecks(nullabilityMap: Map<AbstractInsnNode, Nullability>) {
+        private fun transformTrivialChecks(nullabilityMap: Map<AbstractInsnNode, StrictBasicValue>) {
-            for ((insn, nullability) in nullabilityMap) {
+            for ((insn, value) in nullabilityMap) {
+                val nullability = value.getNullability()
                 when (insn.opcode) {
                     Opcodes.IFNULL -> transformTrivialNullJump(insn as JumpInsnNode, nullability == Nullability.NULL)
                     Opcodes.IFNONNULL -> transformTrivialNullJump(insn as JumpInsnNode, nullability == Nullability.NOT_NULL)
-                    Opcodes.INSTANCEOF -> transformInstanceOf(insn, nullability)
+                    Opcodes.INSTANCEOF -> transformInstanceOf(insn as TypeInsnNode, nullability, value)
                 }
             }
         }
@@ -127,37 +129,22 @@ class RedundantNullCheckMethodTransformer : MethodTransformer() {
             }
         }
 
-        private fun transformInstanceOf(insn: AbstractInsnNode, nullability: Nullability) {
+        private fun transformInstanceOf(insn: TypeInsnNode, nullability: Nullability, value: StrictBasicValue) {
-            if (nullability != Nullability.NULL) return
             if (ReifiedTypeInliner.isOperationReifiedMarker(insn.previous)) return
-
+            if (nullability == Nullability.NULL) {
-            changes = true
+                changes = true
-
+                transformTrivialInstanceOf(insn, false)
-            val nextOpcode = insn.next?.opcode
+            }
-            if (nextOpcode == Opcodes.IFEQ || nextOpcode == Opcodes.IFNE)
+            else if (nullability == Nullability.NOT_NULL && value.type.internalName == insn.desc) {
-                transformNullInstanceOfWithJump(insn)
+                changes = true
-            else
+                transformTrivialInstanceOf(insn, true)
-                transformNullInstanceOf(insn)
-        }
-
-        private fun transformNullInstanceOf(insn: AbstractInsnNode) {
-            methodNode.instructions.run {
-                popReferenceValueBefore(insn)
-                set(insn, InsnNode(Opcodes.ICONST_0))
             }
         }
 
-        private fun transformNullInstanceOfWithJump(insn: AbstractInsnNode) {
+        private fun transformTrivialInstanceOf(insn: AbstractInsnNode, constValue: Boolean) {
             methodNode.instructions.run {
                 popReferenceValueBefore(insn)
-                val jump = insn.next.assertedCast<JumpInsnNode> { "JumpInsnNode expected" }
+                set(insn, if (constValue) InsnNode(Opcodes.ICONST_1) else InsnNode(Opcodes.ICONST_0))
-                remove(insn)
-                if (jump.opcode == Opcodes.IFEQ) {
-                    set(jump, JumpInsnNode(Opcodes.GOTO, jump.label))
-                }
-                else {
-                    remove(jump)
-                }
             }
         }
 
@@ -345,7 +332,9 @@ class RedundantNullCheckMethodTransformer : MethodTransformer() {
 }
 
 internal fun AbstractInsnNode.isInstanceOfOrNullCheck() =
-        opcode == Opcodes.INSTANCEOF || opcode == Opcodes.IFNULL || opcode == Opcodes.IFNONNULL
+        opcode == Opcodes.INSTANCEOF ||
+        opcode == Opcodes.IFNULL ||
+        opcode == Opcodes.IFNONNULL
 
 internal fun AbstractInsnNode.isCheckParameterNotNull() =
         isInsn<MethodInsnNode>(Opcodes.INVOKESTATIC) {

compiler/testData/codegen/box/nullCheckOptimization/trivialInstanceOf.kt

@@ -0,0 +1,17 @@
+sealed class A {
+    class B : A()
+
+    class C : A()
+}
+
+inline fun foo(): A = A.B()
+
+fun box(): String {
+    val a: A = foo()
+    val b: Boolean
+    when (a) {
+        is A.B -> b = true
+        is A.C -> b = false
+    }
+    return if (b) "OK" else "FAIL"
+}

compiler/testData/codegen/bytecodeText/nullCheckOptimization/trivialInstanceOf.kt

@@ -0,0 +1,24 @@
+sealed class A {
+    class B : A()
+
+    class C : A()
+}
+
+inline fun foo(): A = A.B()
+
+fun box(): String {
+    val a: A = foo()
+    val b: Boolean
+    when (a) {
+        is A.B -> b = true
+        is A.C -> b = false
+    }
+    return if (b) "OK" else "FAIL"
+}
+
+// 0 TABLESWITCH
+// 0 LOOKUPSWITCH
+// 0 ATHROW
+// 0 INSTANCEOF
+// 0 FAIL
+// 0 POP

compiler/testData/codegen/bytecodeText/when/sealedWhenInitialization.kt

@@ -4,8 +4,10 @@ sealed class A {
     class C : A()
 }
 
+fun foo(): A = A.C()
+
 fun box(): String {
-    val a: A = A.C()
+    val a: A = foo()
     val b: Boolean
     when (a) {
         A.B -> b = true

compiler/tests-ir-jvm/tests/org/jetbrains/kotlin/codegen/ir/IrBlackBoxCodegenTestGenerated.java

@@ -11443,6 +11443,12 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/kt7774.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("trivialInstanceOf.kt")
+        public void testTrivialInstanceOf() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/trivialInstanceOf.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/box/objectIntrinsics")

compiler/tests/org/jetbrains/kotlin/codegen/BlackBoxCodegenTestGenerated.java

@@ -11443,6 +11443,12 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/kt7774.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("trivialInstanceOf.kt")
+        public void testTrivialInstanceOf() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/trivialInstanceOf.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/box/objectIntrinsics")

compiler/tests/org/jetbrains/kotlin/codegen/BytecodeTextTestGenerated.java

@@ -1792,6 +1792,12 @@ public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/nullCheckOptimization/reifiedNullIs.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("trivialInstanceOf.kt")
+        public void testTrivialInstanceOf() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/nullCheckOptimization/trivialInstanceOf.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/bytecodeText/ranges")

compiler/tests/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -11443,6 +11443,12 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/kt7774.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("trivialInstanceOf.kt")
+        public void testTrivialInstanceOf() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/trivialInstanceOf.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/box/objectIntrinsics")

js/js.tests/test/org/jetbrains/kotlin/js/test/semantics/JsCodegenBoxTestGenerated.java

@@ -12799,6 +12799,12 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
             String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/kt7774.kt");
             doTest(fileName);
         }
+
+        @TestMetadata("trivialInstanceOf.kt")
+        public void testTrivialInstanceOf() throws Exception {
+            String fileName = KotlinTestUtils.navigationMetadata("compiler/testData/codegen/box/nullCheckOptimization/trivialInstanceOf.kt");
+            doTest(fileName);
+        }
     }
 
     @TestMetadata("compiler/testData/codegen/box/objectIntrinsics")