Store stack values before inlines. #KT-5634 Fixed

Inlining code violates JIT assumptions for OSR in Java 8 in case of
starting with non-empty and containing cycles.

Fix is to mark each inlined block with markers and then store/restore
stack state before and after inlined body.
This commit is contained in:
Denis Zharkov
2014-08-21 08:04:50 +04:00
committed by Andrey Breslav
parent 469db95662
commit f1d9d11590
15 changed files with 468 additions and 3 deletions
@@ -36,7 +36,7 @@ public class OptimizationMethodVisitor extends MethodVisitor {
private static final int MEMORY_LIMIT_BY_METHOD_MB = 50;
private static final MethodTransformer[] TRANSFORMERS = new MethodTransformer[]{
new RedundantNullCheckMethodTransformer(), new RedundantBoxingMethodTransformer(),
new RedundantGotoMethodTransformer()
new RedundantGotoMethodTransformer(), new StoreStackBeforeInlineMethodTransformer()
};
private final MethodNode methodNode;
@@ -0,0 +1,218 @@
/*
* Copyright 2010-2014 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.jetbrains.jet.codegen.optimization
import org.jetbrains.jet.codegen.optimization.transformer.MethodTransformer
import org.jetbrains.jet.codegen.optimization.common.OptimizationBasicInterpreter
import org.jetbrains.jet.codegen.inline.InlineCodegenUtil
import org.jetbrains.org.objectweb.asm.tree.MethodNode
import org.jetbrains.org.objectweb.asm.tree.AbstractInsnNode
import org.jetbrains.org.objectweb.asm.Opcodes
import org.jetbrains.org.objectweb.asm.tree.MethodInsnNode
import org.jetbrains.org.objectweb.asm.tree.analysis.Frame
import org.jetbrains.org.objectweb.asm.tree.analysis.BasicValue
import org.jetbrains.org.objectweb.asm.tree.VarInsnNode
import org.jetbrains.org.objectweb.asm.Type
import com.intellij.util.containers.Stack
class StoreStackBeforeInlineMethodTransformer : MethodTransformer() {
override fun transform(internalClassName: String, methodNode: MethodNode) {
val frames = MethodTransformer.analyze(internalClassName, methodNode, OptimizationBasicInterpreter())
if (needToProcess(methodNode, frames)) {
process(methodNode, frames)
}
else {
removeInlineMarkers(methodNode)
}
}
}
private fun needToProcess(node: MethodNode, frames: Array<Frame<BasicValue>?>): Boolean {
val insns = node.instructions.toArray()
var balance = 0
var isThereAnyInlineMarker = false
for ((insn, frame) in insns.zip(frames)) {
if (isInlineMarker(insn)) {
isThereAnyInlineMarker = true
// inline marker is not available
if (frame == null) return false
}
if (isBeforeInlineMarker(insn)) {
balance++
}
else if(isAfterInlineMarker(insn)) {
balance--
}
if (balance < 0) return false
}
return balance == 0 && isThereAnyInlineMarker
}
private fun isBeforeInlineMarker(insn: AbstractInsnNode) = isInlineMarker(insn, InlineCodegenUtil.INLINE_MARKER_BEFORE_METHOD_NAME)
private fun isAfterInlineMarker(insn: AbstractInsnNode) = isInlineMarker(insn, InlineCodegenUtil.INLINE_MARKER_AFTER_METHOD_NAME)
private fun isInlineMarker(insn: AbstractInsnNode, markerName: String? = null): Boolean {
return insn.getOpcode() == Opcodes.INVOKESTATIC &&
insn is MethodInsnNode &&
insn.owner == InlineCodegenUtil.INLINE_MARKER_CLASS_NAME &&
if (markerName != null) markerName == insn.name else true
}
private fun process(methodNode: MethodNode, frames: Array<Frame<BasicValue>?>) {
val insns = methodNode.instructions.toArray()
val storedValuesDescriptorsStack = Stack<StoredStackValuesDescriptor>()
var firstAvailableVarIndex = methodNode.maxLocals
var currentStoredValuesCount = 0
for ((insn, frame) in insns.zip(frames)) {
if (isBeforeInlineMarker(insn)) {
frame ?: throw AssertionError("process method shouldn't be called if frame is null before inline marker")
val desc = storeStackValues(methodNode, frame, insn, firstAvailableVarIndex, currentStoredValuesCount)
firstAvailableVarIndex += desc.storedStackSize
currentStoredValuesCount += desc.storedValuesCount
storedValuesDescriptorsStack.push(desc)
}
else if (isAfterInlineMarker(insn)) {
frame ?: throw AssertionError("process method shouldn't be called if frame is null before inline marker")
val desc = storedValuesDescriptorsStack.pop() ?:
throw AssertionError("should be non null becase markers are balanced")
loadStackValues(methodNode, frame, insn, desc)
firstAvailableVarIndex -= desc.storedStackSize
currentStoredValuesCount -= desc.storedValuesCount
}
if (isInlineMarker(insn)) {
methodNode.instructions.remove(insn)
}
}
}
private class StoredStackValuesDescriptor(
val values: List<BasicValue>,
val firstVariableIndex: Int,
val storedStackSize: Int,
alreadyStoredValuesCount: Int
) {
val nextFreeVarIndex : Int get() = firstVariableIndex + storedStackSize
val storedValuesCount: Int get() = values.size
val isStored: Boolean get() = storedValuesCount > 0
val totalValuesCountOnStackBeforeInline = alreadyStoredValuesCount + storedValuesCount
}
private fun removeInlineMarkers(node: MethodNode) {
for (insn in node.instructions.toArray()) {
if (isInlineMarker(insn)) {
node.instructions.remove(insn)
}
}
}
private fun storeStackValues(
node: MethodNode,
frame: Frame<BasicValue>,
beforeInlineMarker: AbstractInsnNode,
firstAvailableVarIndex: Int,
alreadyStoredValuesCount: Int
) : StoredStackValuesDescriptor {
var stackSize = 0
val values = frame.getStackValuesStartingFrom(alreadyStoredValuesCount)
for (value in values.reverse()) {
node.instructions.insertBefore(
beforeInlineMarker,
VarInsnNode(
value.getType()!!.getOpcode(Opcodes.ISTORE),
firstAvailableVarIndex + stackSize
)
)
stackSize += value.getSize()
}
node.updateMaxLocals(firstAvailableVarIndex + stackSize)
return StoredStackValuesDescriptor(values, firstAvailableVarIndex, stackSize, alreadyStoredValuesCount)
}
private fun loadStackValues(
node: MethodNode,
frame: Frame<BasicValue>,
afterInlineMarker: AbstractInsnNode,
desc: StoredStackValuesDescriptor
) {
if (!desc.isStored) return
val insns = node.instructions
var returnValueVarIndex = -1
var returnType : Type? = null
if (frame.getStackSize() != desc.totalValuesCountOnStackBeforeInline) {
// only returned value
assert(
(frame.getStackSize() - desc.totalValuesCountOnStackBeforeInline) == 1,
"Stack sizes should not differ by more than 1 (returned value)"
)
returnValueVarIndex = desc.nextFreeVarIndex
returnType = frame.getStack(frame.getStackSize() - 1)!!.getType()
node.updateMaxLocals(returnValueVarIndex + returnType!!.getSize())
insns.insertBefore(
afterInlineMarker,
VarInsnNode(returnType!!.getOpcode(Opcodes.ISTORE), returnValueVarIndex)
)
}
var currentVarIndex = desc.firstVariableIndex + desc.storedStackSize
for (value in desc.values) {
currentVarIndex -= value.getSize()
insns.insertBefore(
afterInlineMarker,
VarInsnNode(
value.getType()!!.getOpcode(Opcodes.ILOAD),
currentVarIndex
)
)
}
if (returnValueVarIndex != -1) {
insns.insertBefore(
afterInlineMarker,
VarInsnNode(returnType!!.getOpcode(Opcodes.ILOAD), returnValueVarIndex)
)
}
}
private fun <V : BasicValue> Frame<V>.getStackValuesStartingFrom(from: Int): List<V> =
IntRange(from, getStackSize() - 1).map { getStack(it) }.requireNoNulls()
private fun MethodNode.updateMaxLocals(newMaxLocals: Int) {
maxLocals = Math.max(maxLocals, newMaxLocals)
}
@@ -16,12 +16,15 @@
package org.jetbrains.jet.codegen.optimization.transformer;
import kotlin.jvm.KotlinSignature;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.org.objectweb.asm.tree.MethodNode;
import org.jetbrains.org.objectweb.asm.tree.analysis.*;
public abstract class MethodTransformer {
@KotlinSignature("fun <V : Value?> runAnalyzer(analyzer: Analyzer<V>, internalClassName: String, node: MethodNode): Array<Frame<V>?>")
@NotNull
protected static <V extends Value> Frame<V>[] runAnalyzer(
@NotNull Analyzer<V> analyzer,
@NotNull String internalClassName,
@@ -34,6 +37,9 @@ public abstract class MethodTransformer {
throw new RuntimeException(e);
}
}
@KotlinSignature("fun <V : Value?> analyze(internalClassName: String, node: MethodNode, interpreter: Interpreter<V>): Array<Frame<V>?>")
@NotNull
protected static <V extends Value> Frame<V>[] analyze(
@NotNull String internalClassName,
@NotNull MethodNode node,
@@ -0,0 +1,14 @@
import kotlin.test.assertEquals
inline fun bar(x: Int, y: Long, z: Byte, s: String) = x.toString() + y.toString() + z.toString() + s
fun foobar(x: Int, y: Long, s: String, z: Byte) = x.toString() + y.toString() + s + z.toString()
fun foo() : String {
return foobar(1, 2L, bar(3, 4L, 5.toByte(), "6"), 7.toByte())
}
fun box() : String {
assertEquals("1234567", foo())
return "OK"
}
@@ -0,0 +1,16 @@
import kotlin.test.assertEquals
fun bar() : Boolean = true
fun foobar1(x: Boolean, y: String, z: String) = x.toString() + y + z
fun foobar2(x: Any, y: String, z: String) = x.toString() + y + z
inline fun foo() = "-"
fun box(): String {
val result1 = foobar1(if (1 == 1) true else bar(), foo(), "OK")
val result2 = foobar2(if (1 == 1) "true" else array("false"), foo(), "OK")
assertEquals("true-OK", result1)
assertEquals("true-OK", result2)
return "OK"
}
@@ -0,0 +1,16 @@
import kotlin.test.assertEquals
inline fun bar(x: Int) : Int {
return x
}
fun foobar(x: Int, y: Int, z: Int) = x + y + z
fun foo() : Int {
return foobar(1, bar(2), 3)
}
fun box() : String {
assertEquals(6, foo())
return "OK"
}
@@ -0,0 +1,24 @@
import kotlin.test.assertEquals
inline fun bar(block: () -> String) : String {
return block()
}
inline fun bar2() : String {
return bar { return "def" }
}
fun foobar(x: String, y: String, z: String) = x + y + z
fun foo() : String {
return foobar(
"abc",
bar2(),
"ghi"
)
}
fun box() : String {
assertEquals("abcdefghi", foo())
return "OK"
}
@@ -0,0 +1,13 @@
import kotlin.test.assertEquals
inline fun bar(x: String, block: (String) -> String) = "def" + block(x)
fun foobar(x: String, y: String, z: String) = x + y + z
fun foo() : String {
return foobar("abc", bar("ghi") { x -> x + "jkl" }, "mno")
}
fun box() : String {
assertEquals("abcdefghijklmno", foo())
return "OK"
}
@@ -0,0 +1,16 @@
inline fun bar(x: Int, y: Long, z: Byte, s: String) = x.toString() + y.toString() + z.toString() + s
fun foobar(x: Int, y: Long, s: String, z: Byte) = x.toString() + y.toString() + s + z.toString()
fun foo() : String {
return foobar(1, 2L, bar(3, 4L, 5.toByte(), "6"), 7.toByte())
}
// 3 ISTORE
// 11 ILOAD
// 2 ASTORE
// 8 ALOAD
// 2 LSTORE
// 6 LLOAD
// 1 MAXLOCALS = 9
// 0 InlineMarker
@@ -0,0 +1,16 @@
fun bar() : Boolean = true
fun foobar(x: Boolean, y: String, z: String) = x.toString() + y + z
inline fun foo() = "-"
fun box() {
val result = foobar(if (1 == 1) true else bar(), foo(), "OK")
}
// 1 ISTORE
// 3 ILOAD
// 2 ASTORE
// 7 ALOAD
// 3 MAXLOCALS = 3
// 0 InlineMarker
@@ -0,0 +1,14 @@
inline fun bar(x: Int) : Int {
return x
}
fun foobar(x: Int, y: Int, z: Int) = x + y + z
fun foo() : Int {
return foobar(1, bar(2), 3)
}
// 3 ISTORE
// 11 ILOAD
// 0 InlineMarker
@@ -0,0 +1,21 @@
inline fun bar(block: () -> String) : String {
return block()
}
inline fun bar2() : String {
return bar { return "def" }
}
fun foobar(x: String, y: String, z: String) = x + y + z
fun foo() : String {
return foobar(
"abc",
bar2(),
"ghi"
)
}
// 12 ALOAD
// 0 ASTORE
// 0 InlineMarker
@@ -0,0 +1,11 @@
inline fun bar(x: String, block: (String) -> String) = "def" + block(x)
fun foobar(x: String, y: String, z: String) = x + y + z
fun foo() : String {
return foobar("abc", bar("ghi") { x -> x + "jkl" }, "mno")
}
// 6 ASTORE
// 21 ALOAD
// 1 MAXLOCALS = 5
// 0 InlineMarker
@@ -32,7 +32,7 @@ import java.util.regex.Pattern;
@SuppressWarnings("all")
@TestMetadata("compiler/testData/codegen/bytecodeText")
@TestDataPath("$PROJECT_ROOT")
@InnerTestClasses({BytecodeTextTestGenerated.BoxingOptimization.class, BytecodeTextTestGenerated.Constants.class, BytecodeTextTestGenerated.DirectInvoke.class, BytecodeTextTestGenerated.Statements.class, BytecodeTextTestGenerated.When.class, BytecodeTextTestGenerated.WhenEnumOptimization.class, BytecodeTextTestGenerated.WhenStringOptimization.class})
@InnerTestClasses({BytecodeTextTestGenerated.BoxingOptimization.class, BytecodeTextTestGenerated.Constants.class, BytecodeTextTestGenerated.DirectInvoke.class, BytecodeTextTestGenerated.Statements.class, BytecodeTextTestGenerated.StoreStackBeforeInline.class, BytecodeTextTestGenerated.When.class, BytecodeTextTestGenerated.WhenEnumOptimization.class, BytecodeTextTestGenerated.WhenStringOptimization.class})
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
@TestMetadata("accessorForProtected.kt")
@@ -373,6 +373,46 @@ public class BytecodeTextTestGenerated extends AbstractBytecodeTextTest {
}
@TestMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline")
@TestDataPath("$PROJECT_ROOT")
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
public static class StoreStackBeforeInline extends AbstractBytecodeTextTest {
public void testAllFilesPresentInStoreStackBeforeInline() throws Exception {
JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/codegen/bytecodeText/storeStackBeforeInline"), Pattern.compile("^(.+)\\.kt$"), true);
}
@TestMetadata("differentTypes.kt")
public void testDifferentTypes() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline/differentTypes.kt");
doTest(fileName);
}
@TestMetadata("primitiveMerge.kt")
public void testPrimitiveMerge() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline/primitiveMerge.kt");
doTest(fileName);
}
@TestMetadata("simple.kt")
public void testSimple() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline/simple.kt");
doTest(fileName);
}
@TestMetadata("unreachableMarker.kt")
public void testUnreachableMarker() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline/unreachableMarker.kt");
doTest(fileName);
}
@TestMetadata("withLambda.kt")
public void testWithLambda() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/bytecodeText/storeStackBeforeInline/withLambda.kt");
doTest(fileName);
}
}
@TestMetadata("compiler/testData/codegen/bytecodeText/when")
@TestDataPath("$PROJECT_ROOT")
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
@@ -32,7 +32,7 @@ import java.util.regex.Pattern;
@SuppressWarnings("all")
@TestMetadata("compiler/testData/codegen/boxWithStdlib")
@TestDataPath("$PROJECT_ROOT")
@InnerTestClasses({BlackBoxWithStdlibCodegenTestGenerated.Annotations.class, BlackBoxWithStdlibCodegenTestGenerated.Arrays.class, BlackBoxWithStdlibCodegenTestGenerated.BoxingOptimization.class, BlackBoxWithStdlibCodegenTestGenerated.CallableReference.class, BlackBoxWithStdlibCodegenTestGenerated.Casts.class, BlackBoxWithStdlibCodegenTestGenerated.DataClasses.class, BlackBoxWithStdlibCodegenTestGenerated.DefaultArguments.class, BlackBoxWithStdlibCodegenTestGenerated.Evaluate.class, BlackBoxWithStdlibCodegenTestGenerated.FullJdk.class, BlackBoxWithStdlibCodegenTestGenerated.HashPMap.class, BlackBoxWithStdlibCodegenTestGenerated.JdkAnnotations.class, BlackBoxWithStdlibCodegenTestGenerated.PlatformNames.class, BlackBoxWithStdlibCodegenTestGenerated.PlatformStatic.class, BlackBoxWithStdlibCodegenTestGenerated.Ranges.class, BlackBoxWithStdlibCodegenTestGenerated.Reflection.class, BlackBoxWithStdlibCodegenTestGenerated.Regressions.class, BlackBoxWithStdlibCodegenTestGenerated.Strings.class, BlackBoxWithStdlibCodegenTestGenerated.ToArray.class, BlackBoxWithStdlibCodegenTestGenerated.Vararg.class, BlackBoxWithStdlibCodegenTestGenerated.When.class, BlackBoxWithStdlibCodegenTestGenerated.WhenEnumOptimization.class, BlackBoxWithStdlibCodegenTestGenerated.WhenStringOptimization.class})
@InnerTestClasses({BlackBoxWithStdlibCodegenTestGenerated.Annotations.class, BlackBoxWithStdlibCodegenTestGenerated.Arrays.class, BlackBoxWithStdlibCodegenTestGenerated.BoxingOptimization.class, BlackBoxWithStdlibCodegenTestGenerated.CallableReference.class, BlackBoxWithStdlibCodegenTestGenerated.Casts.class, BlackBoxWithStdlibCodegenTestGenerated.DataClasses.class, BlackBoxWithStdlibCodegenTestGenerated.DefaultArguments.class, BlackBoxWithStdlibCodegenTestGenerated.Evaluate.class, BlackBoxWithStdlibCodegenTestGenerated.FullJdk.class, BlackBoxWithStdlibCodegenTestGenerated.HashPMap.class, BlackBoxWithStdlibCodegenTestGenerated.JdkAnnotations.class, BlackBoxWithStdlibCodegenTestGenerated.PlatformNames.class, BlackBoxWithStdlibCodegenTestGenerated.PlatformStatic.class, BlackBoxWithStdlibCodegenTestGenerated.Ranges.class, BlackBoxWithStdlibCodegenTestGenerated.Reflection.class, BlackBoxWithStdlibCodegenTestGenerated.Regressions.class, BlackBoxWithStdlibCodegenTestGenerated.StoreStackBeforeInline.class, BlackBoxWithStdlibCodegenTestGenerated.Strings.class, BlackBoxWithStdlibCodegenTestGenerated.ToArray.class, BlackBoxWithStdlibCodegenTestGenerated.Vararg.class, BlackBoxWithStdlibCodegenTestGenerated.When.class, BlackBoxWithStdlibCodegenTestGenerated.WhenEnumOptimization.class, BlackBoxWithStdlibCodegenTestGenerated.WhenStringOptimization.class})
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
public class BlackBoxWithStdlibCodegenTestGenerated extends AbstractBlackBoxCodegenTest {
public void testAllFilesPresentInBoxWithStdlib() throws Exception {
@@ -2339,6 +2339,46 @@ public class BlackBoxWithStdlibCodegenTestGenerated extends AbstractBlackBoxCode
}
@TestMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline")
@TestDataPath("$PROJECT_ROOT")
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)
public static class StoreStackBeforeInline extends AbstractBlackBoxCodegenTest {
public void testAllFilesPresentInStoreStackBeforeInline() throws Exception {
JetTestUtils.assertAllTestsPresentByMetadata(this.getClass(), new File("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline"), Pattern.compile("^(.+)\\.kt$"), true);
}
@TestMetadata("differentTypes.kt")
public void testDifferentTypes() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline/differentTypes.kt");
doTestWithStdlib(fileName);
}
@TestMetadata("primitiveMerge.kt")
public void testPrimitiveMerge() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline/primitiveMerge.kt");
doTestWithStdlib(fileName);
}
@TestMetadata("simple.kt")
public void testSimple() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline/simple.kt");
doTestWithStdlib(fileName);
}
@TestMetadata("unreachableMarker.kt")
public void testUnreachableMarker() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline/unreachableMarker.kt");
doTestWithStdlib(fileName);
}
@TestMetadata("withLambda.kt")
public void testWithLambda() throws Exception {
String fileName = JetTestUtils.navigationMetadata("compiler/testData/codegen/boxWithStdlib/storeStackBeforeInline/withLambda.kt");
doTestWithStdlib(fileName);
}
}
@TestMetadata("compiler/testData/codegen/boxWithStdlib/strings")
@TestDataPath("$PROJECT_ROOT")
@RunWith(org.jetbrains.jet.JUnit3RunnerWithInners.class)