JVM_IR more compact safe call chains (almost as old BE)

compiler/backend/src/org/jetbrains/kotlin/codegen/optimization/temporaryVals/TemporaryVariablesEliminationTransformer.kt

@@ -6,6 +6,7 @@
 package org.jetbrains.kotlin.codegen.optimization.temporaryVals
 
 import org.jetbrains.kotlin.codegen.optimization.common.InsnSequence
+import org.jetbrains.kotlin.codegen.optimization.common.isMeaningful
 import org.jetbrains.kotlin.codegen.optimization.common.removeUnusedLocalVariables
 import org.jetbrains.kotlin.codegen.optimization.transformer.MethodTransformer
 import org.jetbrains.kotlin.codegen.state.GenerationState
@@ -21,6 +22,8 @@ class TemporaryVariablesEliminationTransformer(private val state: GenerationStat
     override fun transform(internalClassName: String, methodNode: MethodNode) {
         if (!state.isIrBackend) return
 
+        simplifyTrivialInstructions(methodNode)
+
         val cfg = ControlFlowGraph(methodNode)
         processLabels(cfg)
 
@@ -35,6 +38,42 @@ class TemporaryVariablesEliminationTransformer(private val state: GenerationStat
     }
 
 
+    private fun simplifyTrivialInstructions(methodNode: MethodNode) {
+        val insnList = methodNode.instructions
+        for (insn in insnList.toArray()) {
+            when {
+                insn.matchOpcodes(Opcodes.ILOAD, Opcodes.POP) ||
+                        insn.matchOpcodes(Opcodes.FLOAD, Opcodes.POP) ||
+                        insn.matchOpcodes(Opcodes.ALOAD, Opcodes.POP) -> {
+                    // Remove size 1 LOAD immediately followed by a POP
+                    val popInsn = insn.next
+                    insnList.insert(insn, InsnNode(Opcodes.NOP))
+                    insnList.remove(insn)
+                    insnList.remove(popInsn)
+                }
+                insn.matchOpcodes(Opcodes.DLOAD, Opcodes.POP2) ||
+                        insn.matchOpcodes(Opcodes.LLOAD, Opcodes.POP2) -> {
+                    // Remove size 2 LOAD immediately followed by a POP2
+                    val pop2Insn = insn.next
+                    insnList.insert(insn, InsnNode(Opcodes.NOP))
+                    insnList.remove(insn)
+                    insnList.remove(pop2Insn)
+                }
+            }
+        }
+        for (insn in insnList.toArray()) {
+            // Remove NOPs immediately preceded or immediately followed by an instruction that does something
+            // - not a LINENUMBER, not a LABEL, and not a FRAME.
+            if (insn.opcode == Opcodes.NOP) {
+                val next = insn.next
+                val prev = insn.previous
+                if (next != null && next.isMeaningful || prev != null && prev.isMeaningful) {
+                    insnList.remove(insn)
+                }
+            }
+        }
+    }
+
     private class ControlFlowGraph(val methodNode: MethodNode) {
         private val nonTrivialPredecessors = HashMap<LabelNode, MutableList<AbstractInsnNode>>()
 
@@ -49,6 +88,22 @@ class TemporaryVariablesEliminationTransformer(private val state: GenerationStat
         fun hasNonTrivialPredecessors(label: LabelNode) =
             nonTrivialPredecessors.containsKey(label)
 
+        fun getAllPredecessors(label: LabelNode): List<AbstractInsnNode> {
+            val result = ArrayList<AbstractInsnNode>()
+
+            val trivialPredecessor = label.previous
+            if (trivialPredecessor.opcode != Opcodes.GOTO &&
+                trivialPredecessor.opcode !in Opcodes.IRETURN..Opcodes.RETURN &&
+                trivialPredecessor.opcode != Opcodes.ATHROW
+            ) {
+                result.add(trivialPredecessor)
+            }
+
+            result.addAll(nonTrivialPredecessors[label] ?: emptyList())
+
+            return result
+        }
+
         fun hasSinglePredecessor(label: LabelNode, expectedPredecessor: AbstractInsnNode): Boolean {
             var trivialPredecessor = label.previous
             if (trivialPredecessor.opcode == Opcodes.GOTO ||
@@ -334,12 +389,134 @@ class TemporaryVariablesEliminationTransformer(private val state: GenerationStat
                         continue
                     }
                 }
+            } else if (insn.matchOpcodes(Opcodes.ALOAD, Opcodes.IFNULL)) {
+                // This looks like a start of some safe call chain:
+                //      { ... evaluate receiver ... }
+                //      ASTORE v1
+                //      ALOAD v1        << insn
+                //      IFNULL L
+                //      [ possible first receiver - ALOAD or GETSTATIC ]
+                //      ALOAD v1
+                //      { ... call-1 ... }
+                //      ASTORE v2
+                //      ALOAD v2
+                //      IFNULL L
+                //      [ possible first receiver - ALOAD or GETSTATIC ]
+                //      ALOAD v2
+                //      { ... call-2 ... }
+                //      ...
+                //      ASTORE vN
+                //      ALOAD vN
+                //      IFNULL L
+                //      [ possible first receiver - ALOAD or GETSTATIC ]
+                //      ALOAD vN
+                //      { ... call-N ... }
+                //  L:
+                //      { ... if null ... }
+
+                val ifNull1 = insn.next as JumpInsnNode
+                val ifNullLabel = ifNull1.label
+                val predecessors = cfg.getAllPredecessors(ifNullLabel)
+                if (predecessors.all { isRewritableSafeCallPart(it) }) {
+                    predecessors.forEach { rewriteSafeCallPart(it, insnList) }
+                    insnList.insert(ifNullLabel, InsnNode(Opcodes.POP))
+                    maxStackIncrement = max(maxStackIncrement, 1)
+                    continue
+                }
             }
         }
 
         cfg.methodNode.maxStack += maxStackIncrement
     }
 
+    private fun isRewritableSafeCallPart(branchInsn: AbstractInsnNode): Boolean {
+        val start = branchInsn.previous ?: return false
+        //      ALOAD v1        << start
+        //      IFNULL L
+        //      [ possible first receiver - ALOAD or GETSTATIC ]
+        //      ALOAD v1
+        if (start.matchOpcodes(Opcodes.ALOAD, Opcodes.IFNULL)) {
+            val aLoad1 = start as VarInsnNode
+            val ifNull = start.next as JumpInsnNode
+            val ifNullNext = ifNull.next
+            if (ifNullNext.opcode == Opcodes.ALOAD) {
+                val aLoad2 = ifNullNext as VarInsnNode
+                if (aLoad2.`var` == aLoad1.`var`) return true
+                val aLoad2Next = aLoad2.next
+                if (aLoad2Next.opcode == Opcodes.ALOAD) {
+                    val aLoad3 = aLoad2Next as VarInsnNode
+                    if (aLoad3.`var` == aLoad1.`var`) return true
+                }
+            } else if (ifNullNext.matchOpcodes(Opcodes.GETSTATIC, Opcodes.ALOAD)) {
+                val getStaticInsn = ifNullNext as FieldInsnNode
+                if (Type.getType(getStaticInsn.desc).size != 1) return false
+                val aLoad2 = getStaticInsn.next as VarInsnNode
+                if (aLoad2.`var` == aLoad1.`var`) return true
+            }
+        }
+        return false
+    }
+
+    private fun rewriteSafeCallPart(branchInsn: AbstractInsnNode, insnList: InsnList) {
+        val start = branchInsn.previous
+        //      ALOAD v1        << start
+        //      IFNULL L
+        //      [ possible first receiver - ALOAD or GETSTATIC ]
+        //      ALOAD v1
+        val aLoad1 = start as VarInsnNode
+        val ifNull = start.next as JumpInsnNode
+        val ifNullNext = ifNull.next
+        if (ifNullNext.opcode == Opcodes.ALOAD) {
+            val aLoad2 = ifNullNext as VarInsnNode
+            if (aLoad2.`var` == aLoad1.`var`) {
+                // Rewrite
+                //      ALOAD v1
+                //      IFNULL L
+                //      ALOAD v1
+                // to
+                //      ALOAD v1
+                //      DUP
+                //      IFNULL L
+                insnList.insertBefore(ifNull, InsnNode(Opcodes.DUP))
+                insnList.remove(aLoad2)
+                return
+            } else {
+                val aLoad3 = aLoad2.next as VarInsnNode
+                // Rewrite
+                //      ALOAD v1
+                //      IFNULL L
+                //      ALOAD x
+                //      ALOAD v1
+                // to
+                //      ALOAD v1
+                //      DUP
+                //      IFNULL L
+                //      ALOAD x
+                //      SWAP
+                insnList.insertBefore(ifNull, InsnNode(Opcodes.DUP))
+                insnList.remove(aLoad3)
+                insnList.insert(aLoad2, InsnNode(Opcodes.SWAP))
+                return
+            }
+        } else {
+            // Rewrite
+            //      ALOAD v1
+            //      IFNULL L
+            //      GETSTATIC s
+            //      ALOAD v1
+            // to
+            //      ALOAD v1
+            //      DUP
+            //      IFNULL L
+            //      GETSTATIC s
+            //      SWAP
+            val aLoad2 = ifNullNext.next
+            insnList.insertBefore(ifNull, InsnNode(Opcodes.DUP))
+            insnList.remove(aLoad2)
+            insnList.insert(ifNullNext, InsnNode(Opcodes.SWAP))
+        }
+    }
+
     private fun AbstractInsnNode.matchOpcodes(vararg opcodes: Int): Boolean {
         var insn = this
         for (i in opcodes.indices) {

compiler/fir/fir2ir/tests-gen/org/jetbrains/kotlin/test/runners/codegen/FirBytecodeTextTestGenerated.java

@@ -5448,6 +5448,12 @@ public class FirBytecodeTextTestGenerated extends AbstractFirBytecodeTextTest {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/arrayCompoundAssignment.kt");
         }
 
+        @Test
+        @TestMetadata("notNullReceiversInChain.kt")
+        public void testNotNullReceiversInChain() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/notNullReceiversInChain.kt");
+        }
+
         @Test
         @TestMetadata("safeCallChain1.kt")
         public void testSafeCallChain1() throws Exception {
@@ -5471,6 +5477,18 @@ public class FirBytecodeTextTestGenerated extends AbstractFirBytecodeTextTest {
         public void testSafeCallChainMemberExt2() throws Exception {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallChainMemberExt2.kt");
         }
+
+        @Test
+        @TestMetadata("safeCallElvisSafeCallElvisSomething.kt")
+        public void testSafeCallElvisSafeCallElvisSomething() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallElvisSafeCallElvisSomething.kt");
+        }
+
+        @Test
+        @TestMetadata("safeCallWithElvis.kt")
+        public void testSafeCallWithElvis() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallWithElvis.kt");
+        }
     }
 
     @Nested

compiler/ir/backend.jvm/lower/src/org/jetbrains/kotlin/backend/jvm/JvmLower.kt

@@ -412,8 +412,8 @@ private val jvmFilePhases = listOf(
 
     jvmArgumentNullabilityAssertions,
     toArrayPhase,
+    jvmSafeCallFoldingPhase,
     jvmOptimizationLoweringPhase,
-    ifNullExpressionsFusionPhase,
     additionalClassAnnotationPhase,
     typeOperatorLowering,
     replaceKFunctionInvokeWithFunctionInvokePhase,

compiler/ir/backend.jvm/lower/src/org/jetbrains/kotlin/backend/jvm/lower/JvmOptimizationLowering.kt

@@ -14,19 +14,17 @@ import org.jetbrains.kotlin.backend.common.phaser.makeIrFilePhase
 import org.jetbrains.kotlin.backend.jvm.JvmBackendContext
 import org.jetbrains.kotlin.backend.jvm.JvmLoweredStatementOrigin
 import org.jetbrains.kotlin.backend.jvm.ir.IrInlineScopeResolver
-import org.jetbrains.kotlin.backend.jvm.ir.createJvmIrBuilder
 import org.jetbrains.kotlin.backend.jvm.ir.findInlineCallSites
-import org.jetbrains.kotlin.codegen.AsmUtil
 import org.jetbrains.kotlin.descriptors.Modality
 import org.jetbrains.kotlin.ir.IrStatement
-import org.jetbrains.kotlin.ir.builders.*
+import org.jetbrains.kotlin.ir.builders.irGetField
+import org.jetbrains.kotlin.ir.builders.irSetField
 import org.jetbrains.kotlin.ir.declarations.*
 import org.jetbrains.kotlin.ir.declarations.impl.IrVariableImpl
 import org.jetbrains.kotlin.ir.expressions.*
 import org.jetbrains.kotlin.ir.expressions.impl.IrBlockImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrCallImpl
 import org.jetbrains.kotlin.ir.expressions.impl.IrConstImpl
-import org.jetbrains.kotlin.ir.symbols.IrSymbol
 import org.jetbrains.kotlin.ir.symbols.impl.IrPublicSymbolBase
 import org.jetbrains.kotlin.ir.symbols.impl.IrVariableSymbolImpl
 import org.jetbrains.kotlin.ir.types.*
@@ -73,42 +71,6 @@ class JvmOptimizationLowering(val context: JvmBackendContext) : FileLoweringPass
             else -> null
         }
 
-    private class SafeCallInfo(
-        val scopeSymbol: IrSymbol,
-        val tmpVal: IrVariable,
-        val ifNullBranch: IrBranch,
-        val ifNotNullBranch: IrBranch
-    )
-
-    private fun parseSafeCall(expression: IrExpression): SafeCallInfo? {
-        val block = expression as? IrBlock ?: return null
-        if (block.origin != IrStatementOrigin.SAFE_CALL) return null
-        if (block.statements.size != 2) return null
-        val tmpVal = block.statements[0] as? IrVariable ?: return null
-        val scopeOwner = tmpVal.parent as? IrDeclaration ?: return null
-        val scopeSymbol = scopeOwner.symbol
-        val whenExpr = block.statements[1] as? IrWhen ?: return null
-        if (whenExpr.branches.size != 2) return null
-
-        val ifNullBranch = whenExpr.branches[0]
-        val ifNullBranchCondition = ifNullBranch.condition
-        if (ifNullBranchCondition !is IrCall) return null
-        if (ifNullBranchCondition.symbol != context.irBuiltIns.eqeqSymbol) return null
-        val arg0 = ifNullBranchCondition.getValueArgument(0)
-        if (arg0 !is IrGetValue || arg0.symbol != tmpVal.symbol) return null
-        val arg1 = ifNullBranchCondition.getValueArgument(1)
-        if (arg1 !is IrConst<*> || arg1.value != null) return null
-        val ifNullBranchResult = ifNullBranch.result
-        if (ifNullBranchResult !is IrConst<*> || ifNullBranchResult.value != null) return null
-
-        val ifNotNullBranch = whenExpr.branches[1]
-        return SafeCallInfo(scopeSymbol, tmpVal, ifNullBranch, ifNotNullBranch)
-    }
-
-    private fun IrType.isJvmPrimitive(): Boolean =
-        // TODO get rid of type mapper (take care of '@EnhancedNullability', maybe some other stuff).
-        AsmUtil.isPrimitive(context.typeMapper.mapType(this))
-
     override fun lower(irFile: IrFile) {
         irFile.transformChildren(Transformer(irFile.findInlineCallSites(context)), null)
     }
@@ -136,61 +98,11 @@ class JvmOptimizationLowering(val context: JvmBackendContext) : FileLoweringPass
 
                 if (left.isNullConst() && right is IrConst<*> || right.isNullConst() && left is IrConst<*>)
                     return IrConstImpl.constFalse(expression.startOffset, expression.endOffset, context.irBuiltIns.booleanType)
-
-                if (expression.symbol == context.irBuiltIns.eqeqSymbol) {
-                    if (right.type.isJvmPrimitive()) {
-                        parseSafeCall(left)?.let { return rewriteSafeCallEqeqPrimitive(it, expression) }
-                    }
-                    if (left.type.isJvmPrimitive()) {
-                        parseSafeCall(right)?.let { return rewritePrimitiveEqeqSafeCall(it, expression) }
-                    }
-                }
             }
 
             return expression
         }
 
-        private fun IrBuilderWithScope.ifSafe(safeCall: SafeCallInfo, expr: IrExpression): IrExpression =
-            irBlock(origin = IrStatementOrigin.SAFE_CALL) {
-                +safeCall.tmpVal
-                +irIfThenElse(expr.type, safeCall.ifNullBranch.condition, irFalse(), expr)
-            }
-
-        // Fuse safe call with primitive equality to avoid boxing the primitive. `a?.x == p`:
-        //     { val tmp = a; if (tmp == null) null else tmp.x } == p`
-        // is transformed to:
-        //     { val tmp = a; if (tmp == null) false else tmp.x == p }
-        // Note that the original IR implied that `p` is always evaluated, but the rewritten version
-        // only does so if `a` is not null. This is how the old backend does it, and it's consistent
-        // with `a?.x?.equals(p)`.
-        private fun rewriteSafeCallEqeqPrimitive(safeCall: SafeCallInfo, eqeqCall: IrCall): IrExpression =
-            context.createJvmIrBuilder(safeCall.scopeSymbol).run {
-                ifSafe(safeCall, eqeqCall.apply { putValueArgument(0, safeCall.ifNotNullBranch.result) })
-            }
-
-        // Fuse safe call with primitive equality to avoid boxing the primitive. 'p == a?.x':
-        //     p == { val tmp = a; if (tmp == null) null else tmp.x }
-        // is transformed to:
-        //     { val tmp_p = p; { val tmp = a; if (tmp == null) false else p == tmp } }
-        // Note that `p` is evaluated even if `a` is null, which is again consistent with both the old backend
-        // and `p.equals(a?.x)`.
-        private fun rewritePrimitiveEqeqSafeCall(safeCall: SafeCallInfo, eqeqCall: IrCall): IrExpression =
-            context.createJvmIrBuilder(safeCall.scopeSymbol).run {
-                val primitive = eqeqCall.getValueArgument(0)!!
-                if (primitive.isTrivial()) {
-                    ifSafe(safeCall, eqeqCall.apply { putValueArgument(1, safeCall.ifNotNullBranch.result) })
-                } else {
-                    // The extra block for `p`'s variable is intentional as adding it into the inner block
-                    // would make it no longer look like a safe call to `IfNullExpressionFusionLowering`.
-                    irBlock {
-                        +ifSafe(safeCall, eqeqCall.apply {
-                            putValueArgument(0, irGet(irTemporary(primitive)))
-                            putValueArgument(1, safeCall.ifNotNullBranch.result)
-                        })
-                    }
-                }
-            }
-
         private fun optimizePropertyAccess(expression: IrCall, data: IrDeclaration?): IrExpression {
             val accessor = expression.symbol.owner as? IrSimpleFunction ?: return expression
             if (accessor.modality != Modality.FINAL || accessor.isExternal) return expression
@@ -370,12 +282,6 @@ class JvmOptimizationLowering(val context: JvmBackendContext) : FileLoweringPass
         }
 
         override fun visitContainerExpression(expression: IrContainerExpression, data: IrDeclaration?): IrExpression {
-            val safeCall = parseSafeCall(expression)
-            if (safeCall != null) {
-                // Don't optimize out temporary values for safe calls (yet), so that safe call-based equality checks can be optimized.
-                dontTouchTemporaryVals.add(safeCall.tmpVal)
-            }
-
             if (expression.origin == IrStatementOrigin.WHEN) {
                 // Don't optimize out 'when' subject initialized with a variable,
                 // otherwise we might get somewhat weird debugging behavior.
@@ -508,3 +414,4 @@ class JvmOptimizationLowering(val context: JvmBackendContext) : FileLoweringPass
         }
     }
 }
+

compiler/ir/backend.jvm/lower/src/org/jetbrains/kotlin/backend/jvm/lower/JvmSafeCallChainFoldingLowering.kt

@@ -0,0 +1,398 @@
+/*
+ * Copyright 2010-2021 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+package org.jetbrains.kotlin.backend.jvm.lower
+
+import org.jetbrains.kotlin.backend.common.FileLoweringPass
+import org.jetbrains.kotlin.backend.common.phaser.makeIrFilePhase
+import org.jetbrains.kotlin.backend.jvm.JvmBackendContext
+import org.jetbrains.kotlin.backend.jvm.JvmLoweredStatementOrigin
+import org.jetbrains.kotlin.codegen.AsmUtil
+import org.jetbrains.kotlin.ir.IrBuiltIns
+import org.jetbrains.kotlin.ir.IrStatement
+import org.jetbrains.kotlin.ir.declarations.IrFile
+import org.jetbrains.kotlin.ir.declarations.IrVariable
+import org.jetbrains.kotlin.ir.expressions.*
+import org.jetbrains.kotlin.ir.expressions.impl.*
+import org.jetbrains.kotlin.ir.types.IrType
+import org.jetbrains.kotlin.ir.types.isNullable
+import org.jetbrains.kotlin.ir.util.dump
+import org.jetbrains.kotlin.ir.visitors.IrElementTransformerVoid
+import org.jetbrains.kotlin.ir.visitors.transformChildrenVoid
+
+
+val jvmSafeCallFoldingPhase = makeIrFilePhase(
+    ::JvmSafeCallChainFoldingLowering,
+    name = "JvmSafeCallChainFoldingLowering",
+    description = "Fold safe call chains to more compact forms"
+)
+
+
+class JvmSafeCallChainFoldingLowering(val context: JvmBackendContext) : FileLoweringPass {
+    // Overall idea here is to represent (possibly chained) safe calls as an if-expression in the form:
+    //      if ( { val tmp = <safe_receiver>; tmp != null } )
+    //          <safe_call>
+    //      else
+    //          null
+    // This allows chaining safe calls like 'a?.foo()?.bar()?.qux()':
+    //      if ( { val tmp1 = a; tmp1 != null } &&
+    //           { val tmp2 = tmp1.foo(); tmp2 != null } &&
+    //           { val tmp3 = tmp2.bar(); tmp3 != null }
+    //      )
+    //          tmp3.qux()
+    //      else
+    //          null
+    // This also allows fusing safe calls with elvises (and some other operations).
+
+    override fun lower(irFile: IrFile) {
+        irFile.transformChildrenVoid(Transformer())
+    }
+
+    private val booleanNot = context.irBuiltIns.booleanNotSymbol
+
+    private fun IrExpression.irNot() =
+        IrCallImpl.fromSymbolOwner(startOffset, endOffset, booleanNot).apply {
+            dispatchReceiver = this@irNot
+        }
+
+    private fun irAndAnd(left: IrExpression, right: IrExpression): IrExpression =
+        IrCallImpl.fromSymbolOwner(right.startOffset, right.endOffset, context.irBuiltIns.andandSymbol).apply {
+            putValueArgument(0, left)
+            putValueArgument(1, right)
+        }
+
+    private fun IrExpression.irEqEqNull(): IrExpression =
+        IrCallImpl.fromSymbolOwner(this.startOffset, this.endOffset, context.irBuiltIns.eqeqSymbol).apply {
+            putValueArgument(0, this@irEqEqNull)
+            putValueArgument(1, IrConstImpl.constNull(startOffset, endOffset, context.irBuiltIns.nothingNType))
+        }
+
+    private fun IrExpression.wrapWithBlock(origin: IrStatementOrigin?): IrBlock =
+        IrBlockImpl(this.startOffset, this.endOffset, this.type, origin, listOf(this))
+
+    private fun irTrue(startOffset: Int, endOffset: Int) =
+        IrConstImpl.boolean(startOffset, endOffset, context.irBuiltIns.booleanType, true)
+
+    private fun irFalse(startOffset: Int, endOffset: Int) =
+        IrConstImpl.boolean(startOffset, endOffset, context.irBuiltIns.booleanType, false)
+
+    private fun irValNotNull(startOffset: Int, endOffset: Int, irVariable: IrVariable): IrExpression =
+        if (irVariable.type.isNullable())
+            IrGetValueImpl(startOffset, endOffset, irVariable.symbol).irEqEqNull().irNot()
+        else
+            irTrue(startOffset, endOffset)
+
+    private fun IrType.isJvmPrimitive(): Boolean =
+        // TODO get rid of type mapper (take care of '@EnhancedNullability', maybe some other stuff).
+        AsmUtil.isPrimitive(context.typeMapper.mapType(this))
+
+    private inner class Transformer : IrElementTransformerVoid() {
+        override fun visitBlock(expression: IrBlock): IrExpression {
+            expression.transformChildrenVoid()
+
+            val safeCallInfo = expression.parseSafeCall(context.irBuiltIns)
+            if (safeCallInfo != null) {
+                return foldSafeCall(safeCallInfo)
+            }
+
+            val elvisInfo = expression.parseElvis(context.irBuiltIns)
+            if (elvisInfo != null) {
+                return foldElvis(elvisInfo)
+            }
+
+            // TODO 'as?'
+
+            return expression
+        }
+
+        private fun foldSafeCall(safeCallInfo: SafeCallInfo): IrExpression {
+            // Rewrite a safe call in the form:
+            //      {   // SAFE_CALL
+            //          val tmp = <safe_receiver>
+            //          if (tmp == null)
+            //              null
+            //          else
+            //              <call[tmp]>
+            //      }
+            val safeCallBlock = safeCallInfo.block
+            val startOffset = safeCallBlock.startOffset
+            val endOffset = safeCallBlock.endOffset
+            val safeCallType = safeCallBlock.type
+            val safeCallTmpVal = safeCallInfo.tmpVal
+
+            val tmpValInitializer = safeCallTmpVal.initializer
+            if (tmpValInitializer is IrBlock && tmpValInitializer.origin == JvmLoweredStatementOrigin.FOLDED_SAFE_CALL) {
+                // Chained safe call.
+                // If <safe_receiver> is a FOLDED_SAFE_CALL form, rewrite safe call to:
+                //      {   // FOLDED_SAFE_CALL
+                //          if ( <safe_receiver_condition> && { val tmp = <safe_receiver_result>; tmp != null } )
+                //              <call[tmp]>
+                //          else
+                //              null
+                //      }
+                //    where
+                //      <safe_receiver> =
+                //          {   // FOLDED_SAFE_CALL
+                //              if ( <safe_receiver_condition> )
+                //                  <safe_receiver_result>
+                //              else
+                //                  null
+                //          }
+                val foldedBlock: IrBlock = tmpValInitializer
+                val foldedWhen = foldedBlock.statements[0] as IrWhen
+                val safeReceiverCondition = foldedWhen.branches[0].condition
+                val safeReceiverResult = foldedWhen.branches[0].result
+                safeCallTmpVal.initializer = safeReceiverResult
+                safeCallTmpVal.type = safeReceiverResult.type
+                val foldedConditionPart =
+                    IrCompositeImpl(
+                        startOffset, endOffset, context.irBuiltIns.booleanType, null,
+                        listOf<IrStatement>(
+                            safeCallTmpVal,
+                            irValNotNull(startOffset, endOffset, safeCallTmpVal)
+                        )
+                    )
+                foldedBlock.type = safeCallType
+                foldedWhen.type = safeCallType
+                foldedWhen.branches[0].condition = irAndAnd(safeReceiverCondition, foldedConditionPart)
+                foldedWhen.branches[0].result = safeCallInfo.ifNotNullBranch.result
+                return foldedBlock
+            } else {
+                // Simple safe call.
+                // If <safe_receiver> itself is not a FOLDED_SAFE_CALL form, rewrite safe call to:
+                //      {   // FOLDED_SAFE_CALL
+                //          if ( { val tmp = <safe_receiver>; tmp != null } )
+                //              <call[tmp]>
+                //          else
+                //              null
+                //      }
+
+                val foldedCondition =
+                    IrCompositeImpl(
+                        startOffset, endOffset, context.irBuiltIns.booleanType, null,
+                        listOf<IrStatement>(
+                            safeCallTmpVal,
+                            irValNotNull(startOffset, endOffset, safeCallTmpVal)
+                        )
+                    )
+                val safeCallResult = safeCallInfo.ifNotNullBranch.result
+                val nullResult = safeCallInfo.ifNullBranch.result
+                val foldedWhen = IrWhenImpl(
+                    startOffset, endOffset, safeCallType, JvmLoweredStatementOrigin.FOLDED_SAFE_CALL,
+                    listOf(
+                        IrBranchImpl(startOffset, endOffset, foldedCondition, safeCallResult),
+                        IrBranchImpl(startOffset, endOffset, irTrue(startOffset, endOffset), nullResult)
+                    )
+                )
+                return foldedWhen.wrapWithBlock(JvmLoweredStatementOrigin.FOLDED_SAFE_CALL)
+            }
+        }
+
+        private fun foldElvis(elvisInfo: ElvisInfo): IrExpression {
+            val elvisLhs = elvisInfo.elvisLhs
+            val elvisBlock = elvisInfo.block
+            val startOffset = elvisBlock.startOffset
+            val endOffset = elvisBlock.endOffset
+            val elvisType = elvisBlock.type
+            val elvisTmpVal = elvisInfo.tmpVal
+
+            when {
+                elvisLhs is IrBlock && elvisLhs.origin == JvmLoweredStatementOrigin.FOLDED_SAFE_CALL -> {
+                    // Fold elvis with safe call.
+                    // Given elvis expression:
+                    //      {   // ELVIS
+                    //          val tmp = <elvis_lhs>
+                    //          if (tmp == null)
+                    //              <elvis_rhs>
+                    //          else
+                    //              null
+                    //      }
+                    // where <elvis_lhs> is a folded safe call in the form:
+                    //      {   // FOLDED_SAFE_CALL
+                    //          if ( <safe_call_condition> )
+                    //              <safe_call_result>
+                    //          else
+                    //              null
+                    //      }
+                    // rewrite it to
+                    //      {   // FOLDED_ELVIS
+                    //          if ( <safe_call_condition> && { val tmp = <safe_call_result>; tmp != null } )
+                    //              tmp
+                    //          else
+                    //              <elvis_rhs>
+                    //      }
+
+                    val safeCallWhen = elvisLhs.statements[0] as IrWhen
+                    val safeCallCondition = safeCallWhen.branches[0].condition
+                    val safeCallResult = safeCallWhen.branches[0].result
+                    elvisTmpVal.initializer = safeCallResult
+                    elvisTmpVal.type = safeCallResult.type
+                    val foldedConditionPart =
+                        IrCompositeImpl(
+                            startOffset, endOffset, context.irBuiltIns.booleanType, null,
+                            listOf<IrStatement>(
+                                elvisTmpVal,
+                                irValNotNull(startOffset, endOffset, elvisTmpVal)
+                            )
+                        )
+                    val foldedWhen = IrWhenImpl(
+                        startOffset, endOffset, elvisType, JvmLoweredStatementOrigin.FOLDED_ELVIS,
+                        listOf(
+                            IrBranchImpl(
+                                startOffset, endOffset,
+                                irAndAnd(safeCallCondition, foldedConditionPart),
+                                IrGetValueImpl(startOffset, endOffset, elvisTmpVal.symbol)
+                            ),
+                            IrBranchImpl(
+                                startOffset, endOffset,
+                                irTrue(startOffset, endOffset),
+                                elvisInfo.elvisRhs
+                            )
+                        )
+                    )
+                    return foldedWhen.wrapWithBlock(JvmLoweredStatementOrigin.FOLDED_ELVIS)
+                }
+                elvisLhs is IrBlock && elvisLhs.origin == JvmLoweredStatementOrigin.FOLDED_ELVIS -> {
+                    // Append branches to the inner elvis:
+                    //      val t = { // FOLDED_ELVIS
+                    //          if (...) ...
+                    //          else if ...
+                    //          else <innerElvisRhs>
+                    //      }
+                    //      if (t != null) t else <outerElvisRhs>
+                    //  =>
+                    //      { // FOLDED_ELVIS
+                    //          if (...) ...
+                    //          else if ...
+                    //          else if ( { val t = <innerElvisRhs>; t != null } )
+                    //              t
+                    //          else
+                    //              <outerElvisRhs>
+                    //      }
+                    // TODO maybe we can do somewhat better if we analyze innerElvisRhs as well
+                    val innerElvisWhen = elvisLhs.statements[0] as IrWhen
+                    val innerElvisLastBranch = innerElvisWhen.branches.last()
+                    val innerElvisRhs = innerElvisLastBranch.result
+                    elvisTmpVal.initializer = innerElvisRhs
+                    elvisTmpVal.type = innerElvisRhs.type
+                    val newCondition = IrCompositeImpl(
+                        startOffset, endOffset, context.irBuiltIns.booleanType, null,
+                        listOf(
+                            elvisTmpVal, irValNotNull(startOffset, endOffset, elvisTmpVal)
+                        )
+                    )
+                    innerElvisLastBranch.condition = newCondition
+                    innerElvisLastBranch.result = IrGetValueImpl(startOffset, endOffset, elvisTmpVal.symbol)
+                    innerElvisWhen.branches.add(
+                        IrBranchImpl(
+                            startOffset, endOffset,
+                            irTrue(startOffset, endOffset),
+                            elvisInfo.elvisRhs
+                        )
+                    )
+                    innerElvisWhen.type = elvisType
+                    return innerElvisWhen.wrapWithBlock(JvmLoweredStatementOrigin.FOLDED_ELVIS)
+                }
+                else -> {
+                    return elvisInfo.block
+                }
+            }
+        }
+
+        override fun visitCall(expression: IrCall): IrExpression {
+            expression.transformChildrenVoid()
+
+            if (expression.symbol == context.irBuiltIns.eqeqSymbol) {
+                val startOffset = expression.startOffset
+                val endOffset = expression.endOffset
+
+                val left = expression.getValueArgument(0)
+                    ?: throw AssertionError("No value argument #0: ${expression.dump()}")
+                val right = expression.getValueArgument(1)
+                    ?: throw AssertionError("No value argument #1: ${expression.dump()}")
+                if (left is IrBlock && left.origin == JvmLoweredStatementOrigin.FOLDED_SAFE_CALL && right.type.isJvmPrimitive()) {
+                    val safeCallWhen = left.statements[0] as IrWhen
+                    val safeCallResult = safeCallWhen.branches[0].result
+                    expression.putValueArgument(0, safeCallResult)
+                    safeCallWhen.branches[0].result = expression
+                    safeCallWhen.branches[1].result = irFalse(startOffset, endOffset)
+                    safeCallWhen.type = expression.type
+                    return safeCallWhen.wrapWithBlock(origin = null)
+                }
+                if (right is IrBlock && right.origin == JvmLoweredStatementOrigin.FOLDED_SAFE_CALL && left.type.isJvmPrimitive()) {
+                    val safeCallWhen = right.statements[0] as IrWhen
+                    val safeCallResult = safeCallWhen.branches[0].result
+                    expression.putValueArgument(1, safeCallResult)
+                    safeCallWhen.branches[0].result = expression
+                    safeCallWhen.branches[1].result = irFalse(startOffset, endOffset)
+                    safeCallWhen.type = expression.type
+                    return safeCallWhen.wrapWithBlock(origin = null)
+                }
+            }
+
+            return expression
+        }
+
+    }
+
+}
+
+
+internal class SafeCallInfo(
+    val block: IrBlock,
+    val tmpVal: IrVariable,
+    val ifNullBranch: IrBranch,
+    val ifNotNullBranch: IrBranch
+)
+
+internal fun IrBlock.parseSafeCall(irBuiltIns: IrBuiltIns): SafeCallInfo? {
+    if (this.origin != IrStatementOrigin.SAFE_CALL) return null
+    if (this.statements.size != 2) return null
+    val tmpVal = this.statements[0] as? IrVariable ?: return null
+    val whenExpr = this.statements[1] as? IrWhen ?: return null
+    if (whenExpr.branches.size != 2) return null
+
+    val ifNullBranch = whenExpr.branches[0]
+    val ifNullBranchCondition = ifNullBranch.condition
+    if (ifNullBranchCondition !is IrCall) return null
+    if (ifNullBranchCondition.symbol != irBuiltIns.eqeqSymbol) return null
+    val arg0 = ifNullBranchCondition.getValueArgument(0)
+    if (arg0 !is IrGetValue || arg0.symbol != tmpVal.symbol) return null
+    val arg1 = ifNullBranchCondition.getValueArgument(1)
+    if (arg1 !is IrConst<*> || arg1.value != null) return null
+    val ifNullBranchResult = ifNullBranch.result
+    if (ifNullBranchResult !is IrConst<*> || ifNullBranchResult.value != null) return null
+
+    val ifNotNullBranch = whenExpr.branches[1]
+    return SafeCallInfo(this, tmpVal, ifNullBranch, ifNotNullBranch)
+}
+
+
+internal class ElvisInfo(
+    val block: IrBlock,
+    val tmpVal: IrVariable,
+    val elvisLhs: IrExpression,
+    val elvisRhs: IrExpression
+)
+
+internal fun IrBlock.parseElvis(irBuiltIns: IrBuiltIns): ElvisInfo? {
+    if (this.origin != IrStatementOrigin.ELVIS) return null
+    if (this.statements.size != 2) return null
+    val tmpVal = this.statements[0] as? IrVariable ?: return null
+    val whenExpr = this.statements[1] as? IrWhen ?: return null
+    if (whenExpr.branches.size != 2) return null
+
+    val elvisLhs = tmpVal.initializer ?: return null
+    val ifNullBranch = whenExpr.branches[0]
+    val ifNullBranchCondition = ifNullBranch.condition
+    if (ifNullBranchCondition !is IrCall) return null
+    if (ifNullBranchCondition.symbol != irBuiltIns.eqeqSymbol) return null
+    val arg0 = ifNullBranchCondition.getValueArgument(0)
+    if (arg0 !is IrGetValue || arg0.symbol != tmpVal.symbol) return null
+    val arg1 = ifNullBranchCondition.getValueArgument(1)
+    if (arg1 !is IrConst<*> || arg1.value != null) return null
+    val elvisRhs = ifNullBranch.result
+
+    return ElvisInfo(this, tmpVal, elvisLhs, elvisRhs)
+}

compiler/ir/backend.jvm/src/org/jetbrains/kotlin/backend/jvm/StatementOrigins.kt

@@ -12,4 +12,7 @@ interface JvmLoweredStatementOrigin {
     object DO_WHILE_COUNTER_LOOP: IrStatementOriginImpl("DO_WHILE_COUNTER_LOOP")
     object INLINE_LAMBDA : IrStatementOriginImpl("INLINE_LAMBDA")
     object FAKE_CONTINUATION : IrStatementOriginImpl("FAKE_CONTINUATION")
+
+    object FOLDED_SAFE_CALL : IrStatementOriginImpl("FOLDED_SAFE_CALL")
+    object FOLDED_ELVIS : IrStatementOriginImpl("FOLDED_ELVIS")
 }

compiler/testData/codegen/bytecodeText/boxingOptimization/safeCallWithElvis.kt

@@ -6,9 +6,4 @@ fun check(a : A?) : Int {
 
 // 0 valueOf
 // 0 Value\s\(\)
-
-// JVM_TEMPLATES:
 // 0 ACONST_NULL
-
-// JVM_IR_TEMPLATES:
-// 1 ACONST_NULL

compiler/testData/codegen/bytecodeText/boxingOptimization/safeCallWithElvisMultipleFiles.kt

@@ -10,9 +10,4 @@ fun check(a : A?) : Int {
 
 // 0 valueOf
 // 0 Value\s\(\)
-
-// JVM_TEMPLATES:
 // 0 ACONST_NULL
-
-// JVM_IR_TEMPLATES:
-// 1 ACONST_NULL

compiler/testData/codegen/bytecodeText/forLoop/forInReversed/forInReversedEmptyRangeLiteral.kt

@@ -9,13 +9,13 @@
 
 fun box(): String {
     for (i in (4 .. 1).reversed()) {
-        throw AssertionError("Loop should not be executed")
+        throw AssertionError("Loop over empty Int range should not be executed")
     }
     for (i in (4L .. 1L).reversed()) {
-        throw AssertionError("Loop should not be executed")
+        throw AssertionError("Loop over empty Long range should not be executed")
     }
     for (i in ('D' .. 'A').reversed()) {
-        throw AssertionError("Loop should not be executed")
+        throw AssertionError("Loop over empty Char range should not be executed")
     }
     return "OK"
 }
@@ -29,8 +29,14 @@ fun box(): String {
 // 0 getStep
 
 // JVM_IR_TEMPLATES
+// Int- and Char-based loops are completely elimiated
 // 0 ILOAD
-// 2 ISTORE
+// 0 ISTORE
 // 0 IADD
 // 0 ISUB
 // 0 IINC
+// 3 LLOAD
+// 2 LSTORE
+// 1 LADD
+// 0 LSUB
+// 4 LDC

compiler/testData/codegen/bytecodeText/forLoop/unsigned/forInDownToUIntMinValue.kt

@@ -33,8 +33,8 @@ fun f(a: UInt): Int {
 
 
 // JVM_IR_TEMPLATES
-// 6 ILOAD
+// 5 ILOAD
-// 4 ISTORE
+// 3 ISTORE
 // 0 IADD
 // 0 ISUB
 // 2 IINC

compiler/testData/codegen/bytecodeText/inline/inlineSuspendReifiedNoSpilling.kt

@@ -30,4 +30,4 @@ suspend fun ApplicationCall.test(authenticationService: AuthenticationService) {
 // 0 ILOAD 3
 // 0 ILOAD 2
 // 1 \$i\$f\$receiveJSON I .* 2
-// 3 \$i\$f\$respond I .* 3
+// 2 \$i\$f\$respond I .* 3

compiler/testData/codegen/bytecodeText/inline/linenumberForOneParametersArgumentCall.kt

@@ -10,4 +10,4 @@ inline fun lookAtMe(f: (String) -> Unit) {
     f(a) // Should be no unneeded nops on this line, that might be generated for zero-parameters lambda
 }
 
-// 2 NOP
+// 0 NOP

compiler/testData/codegen/bytecodeText/intConstantNullableSafeCall.kt

@@ -2,8 +2,4 @@ val a : Int? = 10
 
 fun foo() = a?.toString()
 
-// JVM_TEMPLATES
 // 1 IFNULL
-
-// JVM_IR_TEMPLATES
-// 1 IFNONNULL

compiler/testData/codegen/bytecodeText/nullCheckOptimization/safeCallAndElvisChains.kt

@@ -13,6 +13,6 @@ class A(val x: String) {
 // 2 ACONST_NULL
 
 // JVM_IR_TEMPLATES
-// 0 IFNULL
+// 4 IFNULL
-// 4 IFNONNULL
+// 2 IFNONNULL
-// 0 ACONST_NULL
+// 2 ACONST_NULL

compiler/testData/codegen/bytecodeText/temporaryVals/notNullReceiversInChain.kt

@@ -0,0 +1,17 @@
+class A(val b: B)
+class B(val c: C)
+class C(val s: String)
+
+fun test(na: A?) =
+    na?.b?.c?.s
+
+// 1 POP
+// 1 ACONST_NULL
+
+// JVM_IR_TEMPLATES
+// 1 DUP
+// 1 IFNULL
+
+// JVM_TEMPLATES
+// 3 DUP
+// 3 IFNULL

compiler/testData/codegen/bytecodeText/temporaryVals/safeCallElvisSafeCallElvisSomething.kt

@@ -0,0 +1,11 @@
+fun test(a: Any?, b: Any?, c: String) =
+    a?.toString() ?: b?.toString() ?: c
+
+// 2 IFNULL
+// 1 ACONST_NULL
+
+// JVM_IR_TEMPLATES
+// 1 IFNONNULL
+
+// JVM_TEMPLATES
+// 2 IFNONNULL

compiler/testData/codegen/bytecodeText/temporaryVals/safeCallWithElvis.kt

@@ -0,0 +1,17 @@
+fun test(a: Any?) =
+    a?.toString()?.hashCode() ?: 0
+
+// 1 ALOAD
+// 0 ASTORE
+// 0 ISTORE
+// 0 ILOAD
+// 1 POP
+// 0 valueOf
+
+// JVM_IR_TEMPLATES
+// 1 DUP
+// 1 IFNULL
+
+// JVM_TEMPLATES
+// 2 DUP
+// 2 IFNULL

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/BytecodeTextTestGenerated.java

@@ -5304,6 +5304,12 @@ public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/arrayCompoundAssignment.kt");
         }
 
+        @Test
+        @TestMetadata("notNullReceiversInChain.kt")
+        public void testNotNullReceiversInChain() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/notNullReceiversInChain.kt");
+        }
+
         @Test
         @TestMetadata("safeCallChain1.kt")
         public void testSafeCallChain1() throws Exception {
@@ -5327,6 +5333,18 @@ public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
         public void testSafeCallChainMemberExt2() throws Exception {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallChainMemberExt2.kt");
         }
+
+        @Test
+        @TestMetadata("safeCallElvisSafeCallElvisSomething.kt")
+        public void testSafeCallElvisSafeCallElvisSomething() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallElvisSafeCallElvisSomething.kt");
+        }
+
+        @Test
+        @TestMetadata("safeCallWithElvis.kt")
+        public void testSafeCallWithElvis() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallWithElvis.kt");
+        }
     }
 
     @Nested

compiler/tests-common-new/tests-gen/org/jetbrains/kotlin/test/runners/codegen/IrBytecodeTextTestGenerated.java

@@ -5448,6 +5448,12 @@ public class IrBytecodeTextTestGenerated extends AbstractIrBytecodeTextTest {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/arrayCompoundAssignment.kt");
         }
 
+        @Test
+        @TestMetadata("notNullReceiversInChain.kt")
+        public void testNotNullReceiversInChain() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/notNullReceiversInChain.kt");
+        }
+
         @Test
         @TestMetadata("safeCallChain1.kt")
         public void testSafeCallChain1() throws Exception {
@@ -5471,6 +5477,18 @@ public class IrBytecodeTextTestGenerated extends AbstractIrBytecodeTextTest {
         public void testSafeCallChainMemberExt2() throws Exception {
             runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallChainMemberExt2.kt");
         }
+
+        @Test
+        @TestMetadata("safeCallElvisSafeCallElvisSomething.kt")
+        public void testSafeCallElvisSafeCallElvisSomething() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallElvisSafeCallElvisSomething.kt");
+        }
+
+        @Test
+        @TestMetadata("safeCallWithElvis.kt")
+        public void testSafeCallWithElvis() throws Exception {
+            runTest("compiler/testData/codegen/bytecodeText/temporaryVals/safeCallWithElvis.kt");
+        }
     }
 
     @Nested