Refined merging logic #KT-5844 Fixed

If merging primitives of different integral types (e.g. I and Z) return
INT.
Else return UNINITIALIZED_VALUE

compiler/backend/src/org/jetbrains/jet/codegen/optimization/boxing/BoxingInterpreter.java

@@ -211,22 +211,12 @@ public class BoxingInterpreter extends OptimizationBasicInterpreter {
             return v;
         }
 
-        if (v instanceof BoxedBasicValue && w == BasicValue.UNINITIALIZED_VALUE) {
-            return v;
-        }
-
-        if (w instanceof BoxedBasicValue && v == BasicValue.UNINITIALIZED_VALUE) {
-            return w;
-        }
-
         if (v instanceof BoxedBasicValue) {
             onMergeFail((BoxedBasicValue) v);
-            return MIXED_VALUE;
         }
 
         if (w instanceof BoxedBasicValue) {
             onMergeFail((BoxedBasicValue) w);
-            return MIXED_VALUE;
         }
 
         return super.merge(v, w);

compiler/backend/src/org/jetbrains/jet/codegen/optimization/common/OptimizationBasicInterpreter.java

@@ -27,7 +27,6 @@ import org.jetbrains.org.objectweb.asm.tree.analysis.BasicInterpreter;
 import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue;
 
 public class OptimizationBasicInterpreter extends BasicInterpreter {
-    public static final BasicValue MIXED_VALUE = new BasicValue(Type.getObjectType("#"));
     private static final BasicValue BOOLEAN_VALUE = new BasicValue(Type.BOOLEAN_TYPE);
     private static final BasicValue CHAR_VALUE = new BasicValue(Type.CHAR_TYPE);
     private static final BasicValue BYTE_VALUE = new BasicValue(Type.BYTE_TYPE);
@@ -93,7 +92,25 @@ public class OptimizationBasicInterpreter extends BasicInterpreter {
             @NotNull BasicValue v, @NotNull BasicValue w
     ) {
         if (!v.equals(w)) {
-            return MIXED_VALUE;
+            if (v == BasicValue.UNINITIALIZED_VALUE || w == BasicValue.UNINITIALIZED_VALUE) {
+                return BasicValue.UNINITIALIZED_VALUE;
+            }
+
+            // if merge of two references then `lub` is java/lang/Object
+            // arrays also are BasicValues with reference type's
+            if (v.getType().getSort() == Type.OBJECT && w.getType().getSort() == Type.OBJECT) {
+                return BasicValue.REFERENCE_VALUE;
+            }
+
+            assert v.getType().getSort() != Type.ARRAY && w.getType().getSort() != Type.ARRAY : "There should not be arrays";
+
+            // if merge of something can be stored in int var (int, char, boolean, byte, character)
+            if (v.getType().getOpcode(Opcodes.ISTORE) == Opcodes.ISTORE &&
+                w.getType().getOpcode(Opcodes.ISTORE) == Opcodes.ISTORE) {
+                return BasicValue.INT_VALUE;
+            }
+
+            return BasicValue.UNINITIALIZED_VALUE;
         }
         return v;
     }

compiler/testData/codegen/boxWithStdlib/boxingOptimization/kt5844.kt

@@ -0,0 +1,28 @@
+import kotlin.test.assertEquals
+
+fun test1() {
+    val u = when (true) {
+        true -> 42
+        else -> 1.0
+    }
+
+    assertEquals(42, u)
+}
+
+fun test2() {
+    val u = 1L.let {
+        when (it) {
+            is Long -> if (it.toLong() == 2L) it.toLong() else it * 2L // CompilationException
+            else -> it.toDouble()
+        }
+    }
+
+    assertEquals(2L, u)
+}
+
+fun box(): String {
+    test1()
+    test2()
+    return "OK"
+}
+

compiler/tests/org/jetbrains/jet/codegen/generated/BlackBoxWithStdlibCodegenTestGenerated.java

@@ -169,6 +169,12 @@ public class BlackBoxWithStdlibCodegenTestGenerated extends AbstractBlackBoxCode
             doTestWithStdlib(fileName);
         }
         
+        @TestMetadata("kt5844.kt")
+        public void testKt5844() throws Exception {
+            String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/boxingOptimization/kt5844.kt");
+            doTestWithStdlib(fileName);
+        }
+        
         @TestMetadata("nullCheck.kt")
         public void testNullCheck() throws Exception {
             String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/boxingOptimization/nullCheck.kt");