JVM KT-49092 fix stack corruption in redundant boxing elimination

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/BoxingAnalyzer.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.codegen.optimization.boxing
+
+import org.jetbrains.kotlin.codegen.optimization.common.FastMethodAnalyzer
+import org.jetbrains.org.objectweb.asm.tree.MethodNode
+import org.jetbrains.org.objectweb.asm.tree.analysis.AnalyzerException
+import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue
+import org.jetbrains.org.objectweb.asm.tree.analysis.Frame
+import org.jetbrains.org.objectweb.asm.tree.analysis.Interpreter
+
+class BoxingAnalyzer(
+    owner: String,
+    method: MethodNode,
+    private val boxingInterpreter: BoxingInterpreter
+) : FastMethodAnalyzer<BasicValue>(owner, method, boxingInterpreter) {
+    override fun newFrame(nLocals: Int, nStack: Int): Frame<BasicValue> =
+        BoxingFrame(nLocals, nStack)
+
+    private inner class BoxingFrame(nLocals: Int, nStack: Int) : Frame<BasicValue>(nLocals, nStack) {
+        override fun merge(frame: Frame<out BasicValue>, interpreter: Interpreter<BasicValue>): Boolean {
+            if (stackSize != frame.stackSize) {
+                throw AnalyzerException(null, "Incompatible stack heights")
+            }
+
+            var changed = false
+            for (i in 0 until locals) {
+                val local = getLocal(i)
+                val merged = boxingInterpreter.mergeLocalVariableValues(local, frame.getLocal(i))
+                if (local != merged) {
+                    setLocal(i, merged)
+                    changed = true
+                }
+            }
+            for (i in 0 until stackSize) {
+                val onStack = getStack(i)
+                val merged = boxingInterpreter.mergeStackValues(onStack, frame.getStack(i))
+                if (onStack != merged) {
+                    setStack(i, merged)
+                    changed = true
+                }
+            }
+            return changed
+        }
+    }
+}

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/BoxingInterpreter.kt

@@ -29,13 +29,13 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.jvm.AsmTypes
 import org.jetbrains.kotlin.resolve.jvm.JvmPrimitiveType
+import org.jetbrains.kotlin.types.*
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.jetbrains.org.objectweb.asm.Type
 import org.jetbrains.org.objectweb.asm.tree.AbstractInsnNode
 import org.jetbrains.org.objectweb.asm.tree.InsnList
 import org.jetbrains.org.objectweb.asm.tree.MethodInsnNode
 import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue
-import org.jetbrains.kotlin.types.*
 
 open class BoxingInterpreter(
     private val insnList: InsnList,
@@ -148,6 +148,15 @@ open class BoxingInterpreter(
                 value.type != null && isProgressionClass(value.type)
 
     override fun merge(v: BasicValue, w: BasicValue) =
+        mergeStackValues(v, w)
+
+    fun mergeLocalVariableValues(v: BasicValue, w: BasicValue) =
+        merge(v, w, isLocalVariable = true)
+
+    fun mergeStackValues(v: BasicValue, w: BasicValue) =
+        merge(v, w, isLocalVariable = false)
+
+    private fun merge(v: BasicValue, w: BasicValue, isLocalVariable: Boolean) =
         when {
             v == StrictBasicValue.UNINITIALIZED_VALUE || w == StrictBasicValue.UNINITIALIZED_VALUE ->
                 StrictBasicValue.UNINITIALIZED_VALUE
@@ -156,18 +165,32 @@ open class BoxingInterpreter(
                 when {
                     v is TaintedBoxedValue -> v
                     w is TaintedBoxedValue -> w
-                    v.type != w.type -> v.taint()
+                    v.type != w.type -> mergeBoxedHazardous(v, w, isLocalVariable)
                     else -> v
                 }
             }
             v is BoxedBasicValue ->
-                v.taint()
+                mergeBoxedHazardous(v, w, isLocalVariable)
             w is BoxedBasicValue ->
-                w.taint()
+                mergeBoxedHazardous(w, v, isLocalVariable)
             else ->
                 super.merge(v, w)
         }
 
+    private fun mergeBoxedHazardous(boxed: BoxedBasicValue, other: BasicValue, isLocalVariable: Boolean): BasicValue {
+        if (isLocalVariable) {
+            return boxed.taint()
+        }
+
+        // If we merge a boxed stack value with a value of a different type, mark it as merge hazard immediately:
+        // its intended boxed use might be dead code (KT-49092), in which case boxing elimination would produce incompatible stacks.
+        onMergeFail(boxed)
+        if (other is BoxedBasicValue) {
+            onMergeFail(other)
+        }
+        return boxed
+    }
+
     protected open fun onNewBoxedValue(value: BoxedBasicValue) {}
     protected open fun onUnboxing(insn: AbstractInsnNode, value: BoxedBasicValue, resultType: Type) {}
     protected open fun onAreEqual(insn: AbstractInsnNode, value1: BoxedBasicValue, value2: BoxedBasicValue) {}

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/RedundantBoxingMethodTransformer.kt

@@ -42,7 +42,7 @@ class RedundantBoxingMethodTransformer(private val generationState: GenerationSt
             return
 
         val interpreter = RedundantBoxingInterpreter(node.instructions, generationState)
-        val frames = analyze(internalClassName, node, interpreter)
+        val frames = BoxingAnalyzer(internalClassName, node, interpreter).analyze()
 
         interpretPopInstructionsForBoxedValues(interpreter, node, frames)
 

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBlackBoxCodegenTestGenerated.java

@@ -1921,6 +1921,42 @@ public class FirBlackBoxCodegenTestGenerated extends AbstractFirBlackBoxCodegenT
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @Test
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @Test
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {

compiler/testData/codegen/box/boxingOptimization/kt49092a.kt

@@ -0,0 +1,9 @@
+fun foo(x: Any, y: Any) {}
+
+val y = true
+val z = 1L
+
+fun box(): String {
+    var q = "Failed"
+    foo(if (y) { q = "OK"; z } else "", return q)
+}

compiler/testData/codegen/box/boxingOptimization/kt49092b.kt

@@ -0,0 +1,9 @@
+fun foo(x: Any, y: Any) {}
+
+val y = false
+val z = 1L
+
+fun box(): String {
+    var q = "Failed"
+    foo(if (y) "" else { q = "OK"; z }, return q)
+}

compiler/testData/codegen/box/boxingOptimization/kt49092c.kt

@@ -0,0 +1,10 @@
+fun foo(x: Any, y: Any) {}
+
+val y = false
+val zInt = 1
+val zLong = 1L
+
+fun box(): String {
+    var q = "Failed"
+    foo(if (y) zInt else { q = "OK"; zLong }, return q)
+}

compiler/testData/codegen/box/boxingOptimization/kt49092d.kt

@@ -0,0 +1,9 @@
+fun foo(x: Any, y: Any) {}
+
+val y = true
+val z = 1
+
+fun box(): String {
+    var q = "Failed"
+    foo(if (y) { q = "OK"; z } else "", return q)
+}

compiler/testData/codegen/box/boxingOptimization/kt49092e.kt

@@ -0,0 +1,10 @@
+fun foo(x: Any, y: Any) {}
+
+val y = true
+val zByte = 1.toByte()
+val zShort = 1.toShort()
+
+fun box(): String {
+    var q = "Failed"
+    foo(if (y) { q = "OK"; zByte } else zShort, return q)
+}

compiler/testData/codegen/box/boxingOptimization/kt49092f.kt

@@ -0,0 +1,10 @@
+fun foo(x: Any, y: Any) {}
+
+val y = true
+val z = 1L
+
+fun box(): String {
+    var q = "Failed"
+    val v = if (y) { q = "OK"; z } else ""
+    foo(v, return q)
+}

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BlackBoxCodegenTestGenerated.java

@@ -1837,6 +1837,42 @@ public class BlackBoxCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @Test
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @Test
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBlackBoxCodegenTestGenerated.java

@@ -1921,6 +1921,42 @@ public class IrBlackBoxCodegenTestGenerated extends AbstractIrBlackBoxCodegenTes
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @Test
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @Test
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {

compiler/tests-gen/org/jetbrains/kotlin/codegen/LightAnalysisModeTestGenerated.java

@@ -1624,6 +1624,36 @@ public class LightAnalysisModeTestGenerated extends AbstractLightAnalysisModeTes
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {
             runTest("compiler/testData/codegen/box/boxingOptimization/kt49548.kt");

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/JsCodegenBoxTestGenerated.java

@@ -1171,6 +1171,42 @@ public class JsCodegenBoxTestGenerated extends AbstractJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @Test
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @Test
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {

js/js.tests/tests-gen/org/jetbrains/kotlin/js/test/ir/IrJsCodegenBoxTestGenerated.java

@@ -1213,6 +1213,42 @@ public class IrJsCodegenBoxTestGenerated extends AbstractIrJsCodegenBoxTest {
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @Test
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @Test
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @Test
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {

js/js.tests/tests-gen/org/jetbrains/kotlin/js/testOld/wasm/semantics/IrCodegenBoxWasmTestGenerated.java

@@ -1104,6 +1104,36 @@ public class IrCodegenBoxWasmTestGenerated extends AbstractIrCodegenBoxWasmTest
             runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
         }
 
+        @TestMetadata("kt49092a.kt")
+        public void testKt49092a() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+        }
+
+        @TestMetadata("kt49092b.kt")
+        public void testKt49092b() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+        }
+
+        @TestMetadata("kt49092c.kt")
+        public void testKt49092c() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+        }
+
+        @TestMetadata("kt49092d.kt")
+        public void testKt49092d() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+        }
+
+        @TestMetadata("kt49092e.kt")
+        public void testKt49092e() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+        }
+
+        @TestMetadata("kt49092f.kt")
+        public void testKt49092f() throws Exception {
+            runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+        }
+
         @TestMetadata("kt49548.kt")
         public void testKt49548() throws Exception {
             runTest("compiler/testData/codegen/box/boxingOptimization/kt49548.kt");

native/native.tests/tests-gen/org/jetbrains/kotlin/konan/blackboxtest/ExternalTestGenerated.java

@@ -1261,6 +1261,42 @@ public class ExternalTestGenerated extends AbstractExternalNativeBlackBoxTest {
                 runTest("compiler/testData/codegen/box/boxingOptimization/kt48394.kt");
             }
 
+            @Test
+            @TestMetadata("kt49092a.kt")
+            public void testKt49092a() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092a.kt");
+            }
+
+            @Test
+            @TestMetadata("kt49092b.kt")
+            public void testKt49092b() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092b.kt");
+            }
+
+            @Test
+            @TestMetadata("kt49092c.kt")
+            public void testKt49092c() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092c.kt");
+            }
+
+            @Test
+            @TestMetadata("kt49092d.kt")
+            public void testKt49092d() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092d.kt");
+            }
+
+            @Test
+            @TestMetadata("kt49092e.kt")
+            public void testKt49092e() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092e.kt");
+            }
+
+            @Test
+            @TestMetadata("kt49092f.kt")
+            public void testKt49092f() throws Exception {
+                runTest("compiler/testData/codegen/box/boxingOptimization/kt49092f.kt");
+            }
+
             @Test
             @TestMetadata("kt49548.kt")
             public void testKt49548() throws Exception {