diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt index 41b4ed80cf5..8b759fb899d 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/Context.kt @@ -17,6 +17,8 @@ import java.lang.System.out internal final class Context(val config: KonanConfig) : KonanBackendContext() { + val debug = true + var moduleDescriptor: ModuleDescriptor? = null // TODO: make lateinit? diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 3092f1bc544..6eb5fc0c44b 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -17,10 +17,14 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { // TODO: remove, to make CodeGenerator descriptor-agnostic. var constructedClass: ClassDescriptor? = null val vars = VariableManager(this) - var returnSlot: LLVMValueRef? = null - var slotsPhi: LLVMValueRef? = null - var slotCount = 0 - var localAllocs = 0 + private var returnSlot: LLVMValueRef? = null + private var slotsPhi: LLVMValueRef? = null + private var slotCount = 0 + private var localAllocs = 0 + private var arenaSlot: LLVMValueRef? = null + + private val intPtrType = LLVMIntPtrType(llvmTargetData)!! + private val immOneIntPtrType = LLVMConstInt(intPtrType, 1, 1)!! fun prologue(descriptor: FunctionDescriptor) { val llvmFunction = llvmFunction(descriptor) @@ -28,7 +32,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { prologue(llvmFunction, LLVMGetReturnType(getLlvmFunctionType(descriptor))!!) - if (!descriptor.isExported()) { + if (!descriptor.isExported() && !context.debug) { LLVMSetLinkage(llvmFunction, LLVMLinkage.LLVMPrivateLinkage) // (Cannot do this before the function body is created). } @@ -57,6 +61,9 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { // First slot can be assigned to keep pointer to frame local arena. slotCount = 1 localAllocs = 0 + // Is removed by DCE trivially, if not needed. + arenaSlot = intToPtr( + or(ptrToInt(slotsPhi, intPtrType), immOneIntPtrType), kObjHeaderPtrPtr) } fun epilogue() { @@ -90,7 +97,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { val returnPhi = phi(returnType!!) addPhiIncoming(returnPhi, *returns.toList().toTypedArray()) if (returnSlot != null) { - updateLocalRef(returnPhi, returnSlot!!) + updateReturnRef(returnPhi, returnSlot!!) } releaseVars() LLVMBuildRet(builder, returnPhi) @@ -139,6 +146,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun div (arg0: LLVMValueRef, arg1: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildSDiv(builder, arg0, arg1, name)!! fun srem (arg0: LLVMValueRef, arg1: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildSRem(builder, arg0, arg1, name)!! + fun or (arg0: LLVMValueRef, arg1: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildOr (builder, arg0, arg1, name)!! + /* integers comparisons */ fun icmpEq(arg0: LLVMValueRef, arg1: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildICmp(builder, LLVMIntPredicate.LLVMIntEQ, arg0, arg1, name)!! fun icmpGt(arg0: LLVMValueRef, arg1: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildICmp(builder, LLVMIntPredicate.LLVMIntSGT, arg0, arg1, name)!! @@ -154,7 +163,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun bitcast(type: LLVMTypeRef?, value: LLVMValueRef, name: String = "") = LLVMBuildBitCast(builder, value, type, name)!! - fun intToPtr(imm: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, imm, DestTy, Name)!! + fun intToPtr(value: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, value, DestTy, Name)!! + fun ptrToInt(value: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildPtrToInt(builder, value, DestTy, Name)!! fun alloca(type: LLVMTypeRef?, name: String = ""): LLVMValueRef { if (isObjectType(type!!)) { @@ -165,33 +175,13 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { } } - - // Return object slot (ab)used for arena matching given allocation. - private fun arenaSlot() : LLVMValueRef { - return gep(slotsPhi!!, Int32(0).llvm) - } - - fun allocInstance(typeInfo: LLVMValueRef, hint: Int) : LLVMValueRef { - if (hint == SCOPE_FRAME) { - val aux = arenaSlot() - localAllocs++ - return call(context.llvm.arenaAllocInstanceFunction, listOf(typeInfo, aux)) - } else { - val slot = vars.createAnonymousSlot() - return call(context.llvm.allocInstanceFunction, listOf(typeInfo, slot)) - } + fun allocInstance(typeInfo: LLVMValueRef, lifetime: Lifetime) : LLVMValueRef { + return call(context.llvm.allocInstanceFunction, listOf(typeInfo), lifetime) } fun allocArray( - typeInfo: LLVMValueRef, hint: Int, count: LLVMValueRef) : LLVMValueRef { - if (hint == SCOPE_FRAME) { - val aux = arenaSlot() - localAllocs++ - return call(context.llvm.arenaAllocArrayFunction, listOf(typeInfo, count, aux)) - } else { - val slot = vars.createAnonymousSlot() - return call(context.llvm.allocArrayFunction, listOf(typeInfo, count, slot)) - } + typeInfo: LLVMValueRef, count: LLVMValueRef, lifetime: Lifetime) : LLVMValueRef { + return call(context.llvm.allocArrayFunction, listOf(typeInfo, count), lifetime) } fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef { @@ -235,6 +225,10 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { } } + fun updateReturnRef(value: LLVMValueRef, address: LLVMValueRef) { + call(context.llvm.updateReturnRefFunction, listOf(address, value)) + } + // Only use ignoreOld, when sure that memory is freshly inited and have no value. fun updateLocalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) { call(if (ignoreOld) context.llvm.setLocalRefFunction else context.llvm.updateLocalRefFunction, @@ -250,19 +244,44 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { //-------------------------------------------------------------------------// - fun callAtFunctionScope(llvmFunction: LLVMValueRef, args: List, name: String = "") = - call(llvmFunction, args, this::cleanupLandingpad, name) + fun callAtFunctionScope(llvmFunction: LLVMValueRef, args: List, + lifetime: Lifetime) = + call(llvmFunction, args, lifetime, this::cleanupLandingpad) fun call(llvmFunction: LLVMValueRef, args: List, - lazyLandingpad: () -> LLVMBasicBlockRef? = { null }, name: String = ""): LLVMValueRef { + resultLifetime: Lifetime = Lifetime.IRRELEVANT, + lazyLandingpad: () -> LLVMBasicBlockRef? = { null }): LLVMValueRef { + var callArgs = if (isObjectReturn(llvmFunction.type)) { + // If function returns an object - create slot for the returned value or give local arena. + // This allows appropriate rootset accounting by just looking at the stack slots, + // along with ability to allocate in appropriate arena. + val resultSlot = when (resultLifetime.slotType) { + SlotType.ARENA -> { + localAllocs++ + arenaSlot!! + } + SlotType.RETURN -> returnSlot!! + // TODO: for RETURN_IF_ARENA choose between created slot and arenaSlot + // dynamically. + SlotType.ANONYMOUS, SlotType.RETURN_IF_ARENA -> vars.createAnonymousSlot() + else -> throw Error("Incorrect slot type") + } + args + resultSlot + } else { + args + } + return callRaw(llvmFunction, callArgs, lazyLandingpad) + } + private fun callRaw(llvmFunction: LLVMValueRef, args: List, + lazyLandingpad: () -> LLVMBasicBlockRef?): LLVMValueRef { memScoped { val rargs = allocArrayOf(args)[0].ptr if (LLVMIsAFunction(llvmFunction) != null /* the function declaration */ && LLVMAttribute.LLVMNoUnwindAttribute in LLVMGetFunctionAttrSet(llvmFunction)) { - return LLVMBuildCall(builder, llvmFunction, rargs, args.size, name)!! + return LLVMBuildCall(builder, llvmFunction, rargs, args.size, "")!! } else { val landingpad = lazyLandingpad() @@ -276,7 +295,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { } val success = basicBlock() - val result = LLVMBuildInvoke(builder, llvmFunction, rargs, args.size, success, landingpad, name)!! + val result = LLVMBuildInvoke(builder, llvmFunction, rargs, args.size, success, landingpad, "")!! positionAtEnd(success) return result } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index d2ac04ff9a9..29b16df3a71 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -15,11 +15,41 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.module import org.jetbrains.kotlin.types.KotlinType import org.jetbrains.kotlin.types.TypeUtils -// Different scopes/lifetimes of an object, computed by escape analysis. -const val SCOPE_FRAME = 0 -const val SCOPE_GLOBAL = 1 -const val SCOPE_ARENA = 2 -const val SCOPE_PERMANENT = 3 +internal enum class SlotType { + // Frame local arena slot can be used. + ARENA, + // Return slot can be used. + RETURN, + // Return slot, if it is an arena, can be used. + RETURN_IF_ARENA, + // Anonymous slot. + ANONYMOUS, + // Unknown slot type. + UNKNOWN +} + +// Lifetimes class of reference, computed by escape analysis. +internal enum class Lifetime(val slotType: SlotType) { + // If reference is frame-local (only obtained from some call and never leaves). + LOCAL(SlotType.ARENA), + // If reference is only returned. + RETURN_VALUE(SlotType.RETURN), + // If reference is set as field of references of class RETURN_VALUE or INDIRECT_RETURN_VALUE. + INDIRECT_RETURN_VALUE(SlotType.RETURN_IF_ARENA), + // If reference is stored to the field of an incoming parameters. + PARAMETER_FIELD(SlotType.ANONYMOUS), + // If reference refers to the global (either global object or global variable). + GLOBAL(SlotType.ANONYMOUS), + // If reference used to throw. + THROW(SlotType.ANONYMOUS), + // If reference used as an argument of outgoing function. Class can be improved by escape analysis + // of called function. + ARGUMENT(SlotType.ANONYMOUS), + // If reference class is unknown. + UNKNOWN(SlotType.UNKNOWN), + // If reference class is irrelevant. + IRRELEVANT(SlotType.UNKNOWN) +} /** * Provides utility methods to the implementer. @@ -207,10 +237,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { var globalInitIndex:Int = 0 val allocInstanceFunction = importRtFunction("AllocInstance") - val arenaAllocInstanceFunction = importRtFunction("ArenaAllocInstance") val allocArrayFunction = importRtFunction("AllocArrayInstance") - val arenaAllocArrayFunction = importRtFunction("ArenaAllocArrayInstance") val initInstanceFunction = importRtFunction("InitInstance") + val updateReturnRefFunction = importRtFunction("UpdateReturnRef") val setLocalRefFunction = importRtFunction("SetLocalRef") val setGlobalRefFunction = importRtFunction("SetGlobalRef") val updateLocalRefFunction = importRtFunction("UpdateLocalRef") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt index 4348cf577c6..5fa8790731a 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt @@ -2,7 +2,7 @@ package org.jetbrains.kotlin.backend.konan.llvm import org.jetbrains.kotlin.ir.IrElement import org.jetbrains.kotlin.ir.declarations.IrModuleFragment -import org.jetbrains.kotlin.ir.expressions.IrCall +import org.jetbrains.kotlin.ir.expressions.IrMemberAccessExpression import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid import org.jetbrains.kotlin.ir.visitors.acceptVoid @@ -11,14 +11,15 @@ import org.jetbrains.kotlin.ir.visitors.acceptVoid // We build graph with the following nodes: // * allocation set, keeping tuple of [local, ctor call, owner function], AS // * local store set, keeping pair [local, stored], LSS -// * field store set, keeping tuple [local, stored], FSS -// * global store set, [local, stored], GSS +// * field store set, keeping tuple [local, object to store, stored field id], FSS +// * global store set, [local, stored global address], GSS // Function we're trying to compute is the following: // for each element of AS, could it be referred by someone, whose value is // alive on return from function, where element was allocated. // Each element in RS is associated with few elements in AS, which it could refer to. -// TODO: exact algorithm TBD. -internal class EscapeAnalyzerVisitor(val allocHints: MutableMap) : IrElementVisitorVoid { +// +internal class EscapeAnalyzerVisitor( + val lifetimes: MutableMap) : IrElementVisitorVoid { override fun visitElement(element: IrElement) { element.acceptChildrenVoid(this) @@ -29,8 +30,9 @@ internal class EscapeAnalyzerVisitor(val allocHints: MutableMap) : } } -fun prepareAllocHints(irModule: IrModuleFragment, allocHints: MutableMap) { - assert(allocHints.size == 0) +internal fun computeLifetimes(irModule: IrModuleFragment, + lifetimes: MutableMap) { + assert(lifetimes.size == 0) - irModule.acceptVoid(EscapeAnalyzerVisitor(allocHints)) + irModule.acceptVoid(EscapeAnalyzerVisitor(lifetimes)) } \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index de215d6cbcb..7d3e03b103e 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -119,7 +119,7 @@ internal class MetadatorVisitor(val context: Context) : IrElementVisitorVoid { /** * Defines how to generate context-dependent operations. */ -interface CodeContext { +internal interface CodeContext { /** * Generates `return` [value] operation. @@ -132,7 +132,7 @@ interface CodeContext { fun genContinue(destination: IrContinue) - fun genCall(function: LLVMValueRef, args: List): LLVMValueRef + fun genCall(function: LLVMValueRef, args: List, resultLifetime: Lifetime): LLVMValueRef fun genThrow(exception: LLVMValueRef) @@ -168,7 +168,7 @@ interface CodeContext { internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid { val codegen = CodeGenerator(context) - val allocHints = mutableMapOf() + val resultLifetimes = mutableMapOf() //-------------------------------------------------------------------------// @@ -190,7 +190,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid override fun genContinue(destination: IrContinue) = unsupported() - override fun genCall(function: LLVMValueRef, args: List) = unsupported(function) + override fun genCall(function: LLVMValueRef, args: List, resultLifetime: Lifetime) = unsupported(function) override fun genThrow(exception: LLVMValueRef) = unsupported() @@ -237,7 +237,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid override fun visitModuleFragment(module: IrModuleFragment) { context.log("visitModule : ${ir2string(module)}") - prepareAllocHints(module, allocHints) + computeLifetimes(module, resultLifetimes) module.acceptChildrenVoid(this) appendLlvmUsed(context.llvm.usedFunctions) @@ -529,14 +529,14 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid } } - override fun genCall(function: LLVMValueRef, args: List) = - codegen.callAtFunctionScope(function, args) + override fun genCall(function: LLVMValueRef, args: List, resultLifetime: Lifetime) = + codegen.callAtFunctionScope(function, args, resultLifetime) override fun genThrow(exception: LLVMValueRef) { val objHeaderPtr = codegen.bitcast(codegen.kObjHeaderPtr, exception) val args = listOf(objHeaderPtr) - this.genCall(context.llvm.throwExceptionFunction, args) + this.genCall(context.llvm.throwExceptionFunction, args, Lifetime.IRRELEVANT) codegen.unreachable() } @@ -695,7 +695,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val initFunction = value.descriptor.constructors.first { it.valueParameters.size == 0 } val ctor = codegen.llvmFunction(initFunction) val args = listOf(objectPtr, typeInfo, ctor) - val newValue = call(context.llvm.initInstanceFunction, args) + val newValue = call(context.llvm.initInstanceFunction, args, Lifetime.GLOBAL) val bbInitResult = codegen.currentBlock codegen.br(bbExit) @@ -802,7 +802,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid codegen.plus(sum, size!!) } - val array = codegen.allocArray(codegen.typeInfoValue(value.type)!!, SCOPE_GLOBAL, finalLength) + val array = codegen.allocArray(codegen.typeInfoValue(value.type)!!, finalLength, Lifetime.GLOBAL) elements.fold(kImmZero) { sum, (exp, size, isArray) -> if (!isArray) { call(context.llvm.setArrayFunction, listOf(array, sum, exp)) @@ -841,8 +841,10 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid return@map Element(codegen.staticData.kotlinStringLiteral(it as IrConst).llvm, null, string.length) } else { val toStringDescriptor = getToString(it.type) - val string = if (KotlinBuiltIns.isString(it.type)) evaluationResult - else evaluateSimpleFunctionCall(toStringDescriptor, listOf(evaluationResult)) + val string = + if (KotlinBuiltIns.isString(it.type)) evaluationResult + else evaluateSimpleFunctionCall( + toStringDescriptor, listOf(evaluationResult), Lifetime.LOCAL) val length = call(codegen.llvmFunction(kStringLength!!), listOf(string)) return@map Element(string, length, -1) } @@ -856,7 +858,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val constructor = kStringBuilder!!.constructors .firstOrNull { it -> it.valueParameters.size == 1 && KotlinBuiltIns.isInt(it.valueParameters[0].type) }!! - val stringBuilderObj = codegen.allocInstance(codegen.typeInfoValue(kStringBuilder), SCOPE_FRAME) + val stringBuilderObj = codegen.allocInstance(codegen.typeInfoValue(kStringBuilder), Lifetime.LOCAL) call(codegen.llvmFunction(constructor), listOf(stringBuilderObj, totalLength)) @@ -865,7 +867,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid return@fold sum } - return evaluateSimpleFunctionCall(kStringBuilderToString!!, listOf(stringBuilderObj)) + return evaluateSimpleFunctionCall(kStringBuilderToString!!, listOf(stringBuilderObj), + Lifetime.GLOBAL /* TODO: fix */) } //-------------------------------------------------------------------------// @@ -957,8 +960,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid } // The call inside [CatchingScope] must be configured to dispatch exception to the scope's handler. - override fun genCall(function: LLVMValueRef, args: List): LLVMValueRef { - val res = codegen.call(function, args, this::landingpad) + override fun genCall(function: LLVMValueRef, args: List, lifetime: Lifetime): LLVMValueRef { + val res = codegen.call(function, args, lifetime, this::landingpad) return res } @@ -1295,7 +1298,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val srcArg = evaluateExpression(value.argument) // Evaluate src expression. val srcObjInfoPtr = codegen.bitcast(codegen.kObjHeaderPtr, srcArg) // Cast src to ObjInfoPtr. val args = listOf(srcObjInfoPtr, dstTypeInfo) // Create arg list. - call(context.llvm.checkInstanceFunction, args) // Check if dst is subclass of src. + call(context.llvm.checkInstanceFunction, args) // Check if dst is subclass of src. return srcArg } @@ -1632,7 +1635,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid value is IrDelegatingConstructorCall -> return delegatingConstructorCall(value.descriptor, args) - value.descriptor is FunctionDescriptor -> return evaluateFunctionCall(value as IrCall, args) + value.descriptor is FunctionDescriptor -> return evaluateFunctionCall( + value as IrCall, args, resultLifetime(value)) else -> { TODO("${ir2string(value)}") } @@ -1667,11 +1671,12 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid //-------------------------------------------------------------------------// - private fun evaluateFunctionCall(callee: IrCall, args: List): LLVMValueRef { + private fun evaluateFunctionCall(callee: IrCall, args: List, + resultLifetime: Lifetime): LLVMValueRef { val descriptor:FunctionDescriptor = callee.descriptor as FunctionDescriptor if (descriptor.isFunctionInvoke) { - return evaluateFunctionInvoke(descriptor, args) + return evaluateFunctionInvoke(descriptor, args, resultLifetime) } if (descriptor.isIntrinsic) { @@ -1681,7 +1686,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid when (descriptor) { is IrBuiltinOperatorDescriptorBase -> return evaluateOperatorCall (callee, args) is ConstructorDescriptor -> return evaluateConstructorCall (callee, args) - else -> return evaluateSimpleFunctionCall(descriptor, args, callee.superQualifier) + else -> return evaluateSimpleFunctionCall( + descriptor, args, resultLifetime, callee.superQualifier) } } @@ -1696,7 +1702,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid } private fun evaluateFunctionInvoke(descriptor: FunctionDescriptor, - args: List): LLVMValueRef { + args: List, resultLifetime: Lifetime): LLVMValueRef { // Note: the whole function code below is written in the assumption that // `invoke` method receiver is passed as first argument. @@ -1711,22 +1717,24 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid // Get `functionImpl.unboundRef`: val unboundRef = evaluateSimpleFunctionCall(functionImplUnboundRefGetter, - listOf(functionImpl)) + listOf(functionImpl), Lifetime.IRRELEVANT /* unboundRef isn't managed reference */) // Cast `functionImpl.unboundRef` to pointer to function: val entryPtr = codegen.bitcast(pointerType(unboundRefType), unboundRef, "entry") - return call(descriptor, entryPtr, args) + return call(descriptor, entryPtr, args, resultLifetime) } //-------------------------------------------------------------------------// - private fun evaluateSimpleFunctionCall(descriptor: FunctionDescriptor, args: List, superClass: ClassDescriptor? = null): LLVMValueRef { + private fun evaluateSimpleFunctionCall( + descriptor: FunctionDescriptor, args: List, + resultLifetime: Lifetime, superClass: ClassDescriptor? = null): LLVMValueRef { //context.log("evaluateSimpleFunctionCall : $tmpVariableName = ${ir2string(value)}") if (descriptor.isOverridable && superClass == null) - return callVirtual(descriptor, args) + return callVirtual(descriptor, args, resultLifetime) else - return callDirect(descriptor, args) + return callDirect(descriptor, args, resultLifetime) } //-------------------------------------------------------------------------// @@ -1737,12 +1745,13 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid if (descriptor.dispatchReceiverParameter != null) args.add(evaluateExpression(value.dispatchReceiver!!)) //add this ptr args.add(evaluateExpression(value.getValueArgument(0)!!)) - return evaluateSimpleFunctionCall(descriptor, args, value.superQualifier) + return evaluateSimpleFunctionCall( + descriptor, args, Lifetime.IRRELEVANT, value.superQualifier) } //-------------------------------------------------------------------------// - private fun hintForCall(callee: IrCall): Int { - return allocHints.getOrElse(callee) { SCOPE_GLOBAL } + private fun resultLifetime(callee: IrMemberAccessExpression): Lifetime { + return resultLifetimes.getOrElse(callee) { Lifetime.GLOBAL } } private fun evaluateConstructorCall(callee: IrCall, args: List): LLVMValueRef { @@ -1751,12 +1760,13 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val constructedClass = (callee.descriptor as ConstructorDescriptor).constructedClass val thisValue = if (constructedClass.isArray) { assert(args.size >= 1 && args[0].type == int32Type) - codegen.allocArray(codegen.typeInfoValue(constructedClass), hintForCall(callee), args[0]) + codegen.allocArray(codegen.typeInfoValue(constructedClass), args[0], + resultLifetime(callee)) } else { - codegen.allocInstance(codegen.typeInfoValue(constructedClass), hintForCall(callee)) + codegen.allocInstance(codegen.typeInfoValue(constructedClass), resultLifetime(callee)) } evaluateSimpleFunctionCall(callee.descriptor as FunctionDescriptor, - listOf(thisValue) + args) + listOf(thisValue) + args, Lifetime.IRRELEVANT /* constructor doesn't return anything */) return thisValue } } @@ -1851,15 +1861,17 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid //-------------------------------------------------------------------------// - fun callDirect(descriptor: FunctionDescriptor, args: List): LLVMValueRef { + fun callDirect(descriptor: FunctionDescriptor, args: List, + resultLifetime: Lifetime): LLVMValueRef { val realDescriptor = descriptor.resolveFakeOverride().original val llvmFunction = codegen.functionLlvmValue(realDescriptor) - return call(descriptor, llvmFunction, args) + return call(descriptor, llvmFunction, args, resultLifetime) } //-------------------------------------------------------------------------// - fun callVirtual(descriptor: FunctionDescriptor, args: List): LLVMValueRef { + fun callVirtual(descriptor: FunctionDescriptor, args: List, + resultLifetime: Lifetime): LLVMValueRef { val typeInfoPtrPtr = LLVMBuildStructGEP(codegen.builder, args[0], 0 /* type_info */, "")!! val typeInfoPtr = codegen.load(typeInfoPtrPtr) assert (typeInfoPtr.type == codegen.kTypeInfoPtr) @@ -1881,12 +1893,11 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid // for an additional per-interface vtable. val methodHash = codegen.functionHash(descriptor) // Calculate hash of the method to be invoked val lookupArgs = listOf(typeInfoPtr, methodHash) // Prepare args for lookup - call(context.llvm.lookupOpenMethodFunction, - lookupArgs) + call(context.llvm.lookupOpenMethodFunction, lookupArgs) } val functionPtrType = pointerType(codegen.getLlvmFunctionType(descriptor)) // Construct type of the method to be invoked val function = codegen.bitcast(functionPtrType, llvmMethod) // Cast method address to the type - return call(descriptor, function, args) // Invoke the method + return call(descriptor, function, args, resultLifetime) // Invoke the method } //-------------------------------------------------------------------------// @@ -1895,8 +1906,9 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid // instead of a plain list. // In such case it would be possible to check that all args are available and in the correct order. // However, it currently requires some refactoring to be performed. - private fun call(descriptor: FunctionDescriptor, function: LLVMValueRef, args: List): LLVMValueRef { - val result = call(function, args) + private fun call(descriptor: FunctionDescriptor, function: LLVMValueRef, args: List, + resultLifetime: Lifetime): LLVMValueRef { + val result = call(function, args, resultLifetime) if (descriptor.returnType?.isNothing() == true) { codegen.unreachable() } @@ -1908,15 +1920,9 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid return result } - private fun call(function: LLVMValueRef, args: List): LLVMValueRef { - if (codegen.isObjectReturn(function.type)) { - // If function returns an object - create slot for the returned value. - // This allows appropriate rootset accounting by just looking at the stack slots. - val resultSlot = codegen.vars.createAnonymousSlot() - return currentCodeContext.genCall(function, args + resultSlot) - } else { - return currentCodeContext.genCall(function, args) - } + private fun call(function: LLVMValueRef, args: List, + resultLifetime: Lifetime = Lifetime.IRRELEVANT): LLVMValueRef { + return currentCodeContext.genCall(function, args, resultLifetime) } //-------------------------------------------------------------------------// @@ -1934,7 +1940,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid codegen.bitcast(thisPtrArgType, thisPtr) } - return callDirect(descriptor, listOf(thisPtrArg) + args) + return callDirect(descriptor, listOf(thisPtrArg) + args, + Lifetime.IRRELEVANT /* no value returned */) } //-------------------------------------------------------------------------// diff --git a/runtime/src/launcher/cpp/launcher.cpp b/runtime/src/launcher/cpp/launcher.cpp index a077f1f266f..eaf11f0f927 100644 --- a/runtime/src/launcher/cpp/launcher.cpp +++ b/runtime/src/launcher/cpp/launcher.cpp @@ -8,13 +8,13 @@ OBJ_GETTER(setupArgs, int argc, char** argv) { // The count is one less, because we skip argv[0] which is the binary name. - AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT); - ArrayHeader* array = (*OBJ_RESULT)->array(); + ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT); + ArrayHeader* array = result->array(); for (int index = 1; index < argc; index++) { AllocStringInstance(argv[index], strlen(argv[index]), ArrayAddressOfElementAt(array, index - 1)); } - RETURN_OBJ_RESULT(); + return result; } //--- main --------------------------------------------------------------------// diff --git a/runtime/src/main/cpp/Exceptions.cpp b/runtime/src/main/cpp/Exceptions.cpp index 48755a5237d..65fed9011bd 100644 --- a/runtime/src/main/cpp/Exceptions.cpp +++ b/runtime/src/main/cpp/Exceptions.cpp @@ -50,15 +50,13 @@ OBJ_GETTER0(GetCurrentStackTrace) { RuntimeAssert(symbols != nullptr, "Not enough memory to retrieve the stacktrace"); AutoFree autoFree(symbols); - AllocArrayInstance(theArrayTypeInfo, size, OBJ_RESULT); - - ArrayHeader* array = (*OBJ_RESULT)->array(); + ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, size, OBJ_RESULT); + ArrayHeader* array = result->array(); for (int index = 0; index < size; ++index) { AllocStringInstance(symbols[index], strlen(symbols[index]), ArrayAddressOfElementAt(array, index)); } - - RETURN_OBJ_RESULT(); + return result; } void ThrowException(KRef exception) { diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index b5b3f2df7be..6d5f50e61f8 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -90,6 +90,15 @@ inline container_size_t alignUp(container_size_t size, int alignment) { return (size + alignment - 1) & ~(alignment - 1); } +inline bool isArenaSlot(ObjHeader** slot) { + return (reinterpret_cast(slot) & ARENA_BIT) != 0; +} + +inline ObjHeader** asArenaSlot(ObjHeader** slot) { + return reinterpret_cast( + reinterpret_cast(slot) & ~ARENA_BIT); +} + #if USE_GC // Must be vector or map 'container -> number', to keep reference counters correct. @@ -199,7 +208,9 @@ void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) { #endif // USE_GC // We use first slot as place to store frame-local arena container. -ArenaContainer* initedArena(ObjHeader** auxSlot) { +// TODO: create ArenaContainer object on the stack, so that we don't +// do two allocations per frame (ArenaContainer + actual container). +inline ArenaContainer* initedArena(ObjHeader** auxSlot) { ObjHeader* slotValue = *auxSlot; if (slotValue) return reinterpret_cast(slotValue); ArenaContainer* arena = allocMemory(sizeof(ArenaContainer)); @@ -208,6 +219,17 @@ ArenaContainer* initedArena(ObjHeader** auxSlot) { return arena; } +// TODO: shall we do padding for alignment? +inline container_size_t objectSize(const ObjHeader* obj) { + const TypeInfo* type_info = obj->type_info(); + container_size_t size = type_info->instanceSize_ < 0 ? + // An array. + ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader) + : + type_info->instanceSize_ + sizeof(ObjHeader); + return alignUp(size, kObjectAlignment); +} + } // namespace ContainerHeader* AllocContainer(size_t size) { @@ -221,16 +243,7 @@ ContainerHeader* AllocContainer(size_t size) { return result; } -// TODO: shall we do padding for alignment? -uint32_t ObjectSize(const ObjHeader* obj) { - const TypeInfo* type_info = obj->type_info(); - if (type_info->instanceSize_ < 0) { - // An array. - return ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader); - } else { - return type_info->instanceSize_ + sizeof(ObjHeader); - } -} + void FreeContainer(ContainerHeader* header) { RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed"); @@ -262,7 +275,8 @@ void FreeContainer(ContainerHeader* header) { ArrayHeader* array = obj->array(); ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_); } - obj = reinterpret_cast(reinterpret_cast(obj) + ObjectSize(obj)); + obj = reinterpret_cast( + reinterpret_cast(obj) + objectSize(obj)); } // And release underlying memory. @@ -345,6 +359,7 @@ void ArenaContainer::Deinit() { bool ArenaContainer::allocContainer(container_size_t minSize) { auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk); size = alignUp(size, kContainerAlignment); + // TODO: keep simple cache of container chunks. ContainerChunk* result = allocMemory(size); RuntimeAssert(result != nullptr, "Cannot alloc memory"); if (result == nullptr) return false; @@ -493,24 +508,29 @@ void DeinitMemory() { memoryState = nullptr; } -ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) { - RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - return initedArena(auxSlot)->PlaceObject(type_info); -} - OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); + if (isArenaSlot(OBJ_RESULT)) { + auto arena = initedArena(asArenaSlot(OBJ_RESULT)); + auto result = arena->PlaceObject(type_info); +#if TRACE_MEMORY + fprintf(stderr, "instace %p in arena: %p\n", result, arena); +#endif + return result; + } RETURN_OBJ(ObjectContainer(type_info).GetPlace()); } -ObjHeader* ArenaAllocArrayInstance( - const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) { - RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); - return initedArena(auxSlot)->PlaceArray(type_info, elements)->obj(); -} - OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); + if (isArenaSlot(OBJ_RESULT)) { + auto arena = initedArena(asArenaSlot(OBJ_RESULT)); + auto result = arena->PlaceArray(type_info, elements)->obj(); +#if TRACE_MEMORY + fprintf(stderr, "array[%d] %p in arena: %p\n", elements, result, arena); +#endif + return result; + } RETURN_OBJ(ArrayContainer(type_info, elements).GetPlace()->obj()); } @@ -550,7 +570,7 @@ OBJ_GETTER(InitInstance, #if TRACE_MEMORY memoryState->globalObjects->push_back(location); #endif - RETURN_OBJ_RESULT(); + return object; } catch (...) { UpdateLocalRef(OBJ_RESULT, nullptr); UpdateGlobalRef(location, nullptr); @@ -581,7 +601,25 @@ void SetGlobalRef(ObjHeader** location, const ObjHeader* object) { #endif } +void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { + if (isArenaSlot(returnSlot)) return; + ObjHeader* old = *returnSlot; +#if TRACE_MEMORY + fprintf(stderr, "UpdateReturnRef *%p: %p -> %p\n", returnSlot, old, object); +#endif + if (old != object) { + if (object != nullptr) { + AddRef(object); + } + *const_cast(returnSlot) = object; + if (old > reinterpret_cast(1)) { + ReleaseRef(old); + } + } +} + void UpdateLocalRef(ObjHeader** location, const ObjHeader* object) { + RuntimeAssert(!isArenaSlot(location), "must not be a slot"); ObjHeader* old = *location; #if TRACE_MEMORY fprintf(stderr, "UpdateLocalRef *%p: %p -> %p\n", location, old, object); @@ -598,6 +636,7 @@ void UpdateLocalRef(ObjHeader** location, const ObjHeader* object) { } void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) { + RuntimeAssert(!isArenaSlot(location), "Must not be an arena"); #if CONCURRENT ObjHeader* old = *location; #if TRACE_MEMORY @@ -625,9 +664,15 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) { } void LeaveFrame(ObjHeader** start, int count) { +#if TRACE_MEMORY + fprintf(stderr, "LeaveFrame %p .. %p\n", start, start + count); +#endif ReleaseLocalRefs(start + 1, count - 1); if (*start != nullptr) { auto arena = initedArena(start); +#if TRACE_MEMORY + fprintf(stderr, "LeaveFrame: free arena %p\n", arena); +#endif arena->Deinit(); freeMemory(arena); } diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 7db58109722..fe21299e41f 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -5,17 +5,6 @@ #include "Common.h" #include "TypeInfo.h" -typedef enum { - // Allocation guaranteed to be frame local. - SCOPE_FRAME = 0, - // Allocation is generic global allocation. - SCOPE_GLOBAL = 1, - // Allocation shall take place in current stack arena. - SCOPE_ARENA = 2, - // Allocation is permanent. - SCOPE_PERMANENT = 3 -} PlacementHint; - // Must fit in two bits. typedef enum { // Container is normal thread local container. @@ -297,13 +286,23 @@ class ArenaContainer { extern "C" { #endif +// Bit or'ed to slot pointer, marking the fact that allocation shall happen +// in arena pointed by the slot. +#define ARENA_BIT 1 #define OBJ_RESULT __result__ #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT) #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT) -#define RETURN_OBJ(value) { ObjHeader* obj = value; UpdateLocalRef(OBJ_RESULT, obj); return obj; } -#define RETURN_OBJ_RESULT() return *OBJ_RESULT; -#define RETURN_RESULT_OF0(name) name(OBJ_RESULT); return *OBJ_RESULT; -#define RETURN_RESULT_OF(name, ...) name(__VA_ARGS__, OBJ_RESULT); return *OBJ_RESULT; +#define RETURN_OBJ(value) { ObjHeader* obj = value; \ + UpdateReturnRef(OBJ_RESULT, obj); \ + return obj; } +#define RETURN_RESULT_OF0(name) { \ + ObjHeader* obj = name(OBJ_RESULT); \ + return obj; \ + } +#define RETURN_RESULT_OF(name, ...) { \ + ObjHeader* result = name(__VA_ARGS__, OBJ_RESULT); \ + return result; \ + } void InitMemory(); void DeinitMemory(); @@ -320,10 +319,7 @@ void DeinitMemory(); // Escape analysis algorithm is the provider of information for decision on exact aux slot // selection, and comes from upper bound esteemation of object lifetime. // -ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) RUNTIME_NOTHROW; OBJ_GETTER(AllocInstance, const TypeInfo* type_info) RUNTIME_NOTHROW; -ObjHeader* ArenaAllocArrayInstance( - const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) RUNTIME_NOTHROW; OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) RUNTIME_NOTHROW; OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) RUNTIME_NOTHROW; OBJ_GETTER(InitInstance, @@ -359,9 +355,10 @@ void SetGlobalRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW void UpdateLocalRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; // Update potentially globally visible location. void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; +// Update reference in return slot. +void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW; // Optimization: release all references in range. void ReleaseLocalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; -void ReleaseGlobalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; // Called on frame leave, if it has object slots. void LeaveFrame(ObjHeader** start, int count) RUNTIME_NOTHROW; // Collect garbage, which cannot be found by reference counting (cycles). diff --git a/runtime/src/main/cpp/ToString.cpp b/runtime/src/main/cpp/ToString.cpp index ab42153d84a..89979d1cd4c 100644 --- a/runtime/src/main/cpp/ToString.cpp +++ b/runtime/src/main/cpp/ToString.cpp @@ -11,8 +11,8 @@ namespace { OBJ_GETTER(makeString, const char* cstring) { uint32_t length = strlen(cstring); - ArrayHeader* result = ArrayContainer( - theStringTypeInfo, length).GetPlace(); + ArrayHeader* result = AllocArrayInstance( + theStringTypeInfo, length, OBJ_RESULT)->array(); memcpy( ByteArrayAddressOfElementAt(result, 0), cstring,