JVM KT-49092 fix stack corruption in redundant boxing elimination

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/BoxingAnalyzer.kt

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.codegen.optimization.boxing
+
+import org.jetbrains.kotlin.codegen.optimization.common.FastMethodAnalyzer
+import org.jetbrains.org.objectweb.asm.tree.MethodNode
+import org.jetbrains.org.objectweb.asm.tree.analysis.AnalyzerException
+import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue
+import org.jetbrains.org.objectweb.asm.tree.analysis.Frame
+import org.jetbrains.org.objectweb.asm.tree.analysis.Interpreter
+
+class BoxingAnalyzer(
+    owner: String,
+    method: MethodNode,
+    private val boxingInterpreter: BoxingInterpreter
+) : FastMethodAnalyzer<BasicValue>(owner, method, boxingInterpreter) {
+    override fun newFrame(nLocals: Int, nStack: Int): Frame<BasicValue> =
+        BoxingFrame(nLocals, nStack)
+
+    private inner class BoxingFrame(nLocals: Int, nStack: Int) : Frame<BasicValue>(nLocals, nStack) {
+        override fun merge(frame: Frame<out BasicValue>, interpreter: Interpreter<BasicValue>): Boolean {
+            if (stackSize != frame.stackSize) {
+                throw AnalyzerException(null, "Incompatible stack heights")
+            }
+
+            var changed = false
+            for (i in 0 until locals) {
+                val local = getLocal(i)
+                val merged = boxingInterpreter.mergeLocalVariableValues(local, frame.getLocal(i))
+                if (local != merged) {
+                    setLocal(i, merged)
+                    changed = true
+                }
+            }
+            for (i in 0 until stackSize) {
+                val onStack = getStack(i)
+                val merged = boxingInterpreter.mergeStackValues(onStack, frame.getStack(i))
+                if (onStack != merged) {
+                    setStack(i, merged)
+                    changed = true
+                }
+            }
+            return changed
+        }
+    }
+}

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/BoxingInterpreter.kt

@@ -29,13 +29,13 @@ import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 import org.jetbrains.kotlin.resolve.jvm.AsmTypes
 import org.jetbrains.kotlin.resolve.jvm.JvmPrimitiveType
+import org.jetbrains.kotlin.types.*
 import org.jetbrains.org.objectweb.asm.Opcodes
 import org.jetbrains.org.objectweb.asm.Type
 import org.jetbrains.org.objectweb.asm.tree.AbstractInsnNode
 import org.jetbrains.org.objectweb.asm.tree.InsnList
 import org.jetbrains.org.objectweb.asm.tree.MethodInsnNode
 import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue
-import org.jetbrains.kotlin.types.*
 
 open class BoxingInterpreter(
     private val insnList: InsnList,
@@ -148,6 +148,15 @@ open class BoxingInterpreter(
                 value.type != null && isProgressionClass(value.type)
 
     override fun merge(v: BasicValue, w: BasicValue) =
+        mergeStackValues(v, w)
+
+    fun mergeLocalVariableValues(v: BasicValue, w: BasicValue) =
+        merge(v, w, isLocalVariable = true)
+
+    fun mergeStackValues(v: BasicValue, w: BasicValue) =
+        merge(v, w, isLocalVariable = false)
+
+    private fun merge(v: BasicValue, w: BasicValue, isLocalVariable: Boolean) =
         when {
             v == StrictBasicValue.UNINITIALIZED_VALUE || w == StrictBasicValue.UNINITIALIZED_VALUE ->
                 StrictBasicValue.UNINITIALIZED_VALUE
@@ -156,18 +165,32 @@ open class BoxingInterpreter(
                 when {
                     v is TaintedBoxedValue -> v
                     w is TaintedBoxedValue -> w
-                    v.type != w.type -> v.taint()
+                    v.type != w.type -> mergeBoxedHazardous(v, w, isLocalVariable)
                     else -> v
                 }
             }
             v is BoxedBasicValue ->
-                v.taint()
+                mergeBoxedHazardous(v, w, isLocalVariable)
             w is BoxedBasicValue ->
-                w.taint()
+                mergeBoxedHazardous(w, v, isLocalVariable)
             else ->
                 super.merge(v, w)
         }
 
+    private fun mergeBoxedHazardous(boxed: BoxedBasicValue, other: BasicValue, isLocalVariable: Boolean): BasicValue {
+        if (isLocalVariable) {
+            return boxed.taint()
+        }
+
+        // If we merge a boxed stack value with a value of a different type, mark it as merge hazard immediately:
+        // its intended boxed use might be dead code (KT-49092), in which case boxing elimination would produce incompatible stacks.
+        onMergeFail(boxed)
+        if (other is BoxedBasicValue) {
+            onMergeFail(other)
+        }
+        return boxed
+    }
+
     protected open fun onNewBoxedValue(value: BoxedBasicValue) {}
     protected open fun onUnboxing(insn: AbstractInsnNode, value: BoxedBasicValue, resultType: Type) {}
     protected open fun onAreEqual(insn: AbstractInsnNode, value1: BoxedBasicValue, value2: BoxedBasicValue) {}

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/boxing/RedundantBoxingMethodTransformer.kt

@@ -42,7 +42,7 @@ class RedundantBoxingMethodTransformer(private val generationState: GenerationSt
             return
 
         val interpreter = RedundantBoxingInterpreter(node.instructions, generationState)
-        val frames = analyze(internalClassName, node, interpreter)
+        val frames = BoxingAnalyzer(internalClassName, node, interpreter).analyze()
 
         interpretPopInstructionsForBoxedValues(interpreter, node, frames)