From 66293cb89f42fee6992df20820291faacdc85559 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 24 Jan 2017 17:16:54 +0300 Subject: [PATCH] Frame-local allocations. (#193) --- .../backend/konan/llvm/CodeGenerator.kt | 50 +++- .../kotlin/backend/konan/llvm/ContextUtils.kt | 12 +- .../backend/konan/llvm/EscapeAnalysis.kt | 36 +++ .../kotlin/backend/konan/llvm/IrToBitcode.kt | 56 ++-- .../kotlin/backend/konan/llvm/LlvmUtils.kt | 4 +- runtime/src/launcher/cpp/launcher.cpp | 4 +- runtime/src/main/cpp/Exceptions.cpp | 7 +- runtime/src/main/cpp/Memory.cpp | 274 ++++++++++++------ runtime/src/main/cpp/Memory.h | 157 +++++----- runtime/src/main/cpp/Operator.cpp | 2 +- runtime/src/main/kotlin/kotlin/Arrays.kt | 4 +- 11 files changed, 400 insertions(+), 206 deletions(-) create mode 100644 backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index a92ed8f3e67..d98d465d1b7 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -20,6 +20,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { var returnSlot: LLVMValueRef? = null var slotsPhi: LLVMValueRef? = null var slotCount = 0 + var localAllocs = 0 fun prologue(descriptor: FunctionDescriptor) { prologue(llvmFunction(descriptor), @@ -45,16 +46,18 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { cleanupLandingpad = LLVMAppendBasicBlock(function, "cleanup_landingpad")!! positionAtEnd(entryBb!!) slotsPhi = phi(kObjHeaderPtrPtr) - slotCount = 0 + // First slot can be assigned to keep pointer to frame local arena. + slotCount = 1 + localAllocs = 0 } fun epilogue() { appendingTo(prologueBb!!) { - val slots = if (slotCount > 0) + val slots = if (needSlots) LLVMBuildArrayAlloca(builder, kObjHeaderPtr, Int32(slotCount).llvm, "")!! else kNullObjHeaderPtrPtr - if (slotCount > 0) { + if (needSlots) { // Zero-init slots. val slotsMem = bitcast(kInt8Ptr, slots) val pointerSize = LLVMABISizeOfType(llvmTargetData, kObjHeaderPtr).toInt() @@ -102,9 +105,14 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { slotsPhi = null } - fun releaseVars() { - if (slotCount > 0) { - call(context.llvm.releaseLocalRefsFunction, + private val needSlots: Boolean + get() { + return slotCount > 1 || localAllocs > 0 + } + + private fun releaseVars() { + if (needSlots) { + call(context.llvm.leaveFrameFunction, listOf(slotsPhi!!, Int32(slotCount).llvm)) } } @@ -148,6 +156,36 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { return LLVMBuildAlloca(builder, type, name)!! } } + + + // Return object slot (ab)used for arena matching given allocation. + private fun arenaSlot() : LLVMValueRef { + return gep(slotsPhi!!, Int32(0).llvm) + } + + fun allocInstance(typeInfo: LLVMValueRef, hint: Int) : LLVMValueRef { + if (hint == SCOPE_FRAME) { + val aux = arenaSlot() + localAllocs++ + return call(context.llvm.arenaAllocInstanceFunction, listOf(typeInfo, aux)) + } else { + val slot = vars.createAnonymousSlot() + return call(context.llvm.allocInstanceFunction, listOf(typeInfo, slot)) + } + } + + fun allocArray( + typeInfo: LLVMValueRef, hint: Int, count: LLVMValueRef) : LLVMValueRef { + if (hint == SCOPE_FRAME) { + val aux = arenaSlot() + localAllocs++ + return call(context.llvm.arenaAllocArrayFunction, listOf(typeInfo, count, aux)) + } else { + val slot = vars.createAnonymousSlot() + return call(context.llvm.allocArrayFunction, listOf(typeInfo, count, slot)) + } + } + fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef { val result = LLVMBuildLoad(builder, value, name)!! // Use loadSlot() API for that. diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 877e0ef6f0e..43e6191393e 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -24,6 +24,12 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.getSuperClassNotAny import org.jetbrains.kotlin.types.KotlinType import org.jetbrains.kotlin.types.TypeUtils +// Different scopes/lifetimes of an object, computed by escape analysis. +const val SCOPE_FRAME = 0 +const val SCOPE_GLOBAL = 1 +const val SCOPE_ARENA = 2 +const val SCOPE_PERMANENT = 3 + /** * Provides utility methods to the implementer. */ @@ -263,13 +269,15 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { var globalInitIndex:Int = 0 val allocInstanceFunction = importRtFunction("AllocInstance") - val initInstanceFunction = importRtFunction("InitInstance") + val arenaAllocInstanceFunction = importRtFunction("ArenaAllocInstance") val allocArrayFunction = importRtFunction("AllocArrayInstance") + val arenaAllocArrayFunction = importRtFunction("ArenaAllocArrayInstance") + val initInstanceFunction = importRtFunction("InitInstance") val setLocalRefFunction = importRtFunction("SetLocalRef") val setGlobalRefFunction = importRtFunction("SetGlobalRef") val updateLocalRefFunction = importRtFunction("UpdateLocalRef") val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef") - val releaseLocalRefsFunction = importRtFunction("ReleaseLocalRefs") + val leaveFrameFunction = importRtFunction("LeaveFrame") val setArrayFunction = importRtFunction("Kotlin_Array_set") val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl") val lookupFieldOffset = importRtFunction("LookupFieldOffset") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt new file mode 100644 index 00000000000..4348cf577c6 --- /dev/null +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt @@ -0,0 +1,36 @@ +package org.jetbrains.kotlin.backend.konan.llvm + +import org.jetbrains.kotlin.ir.IrElement +import org.jetbrains.kotlin.ir.declarations.IrModuleFragment +import org.jetbrains.kotlin.ir.expressions.IrCall +import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid +import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid +import org.jetbrains.kotlin.ir.visitors.acceptVoid + +// Analysis we're implementing here is as following. +// We build graph with the following nodes: +// * allocation set, keeping tuple of [local, ctor call, owner function], AS +// * local store set, keeping pair [local, stored], LSS +// * field store set, keeping tuple [local, stored], FSS +// * global store set, [local, stored], GSS +// Function we're trying to compute is the following: +// for each element of AS, could it be referred by someone, whose value is +// alive on return from function, where element was allocated. +// Each element in RS is associated with few elements in AS, which it could refer to. +// TODO: exact algorithm TBD. +internal class EscapeAnalyzerVisitor(val allocHints: MutableMap) : IrElementVisitorVoid { + + override fun visitElement(element: IrElement) { + element.acceptChildrenVoid(this) + } + + override fun visitModuleFragment(module: IrModuleFragment) { + module.acceptChildrenVoid(this) + } +} + +fun prepareAllocHints(irModule: IrModuleFragment, allocHints: MutableMap) { + assert(allocHints.size == 0) + + irModule.acceptVoid(EscapeAnalyzerVisitor(allocHints)) +} \ No newline at end of file diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index dd41c2912aa..783d8af129b 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -172,6 +172,7 @@ interface CodeContext { internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid { val codegen = CodeGenerator(context) + val allocHints = mutableMapOf() //-------------------------------------------------------------------------// @@ -240,6 +241,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid override fun visitModuleFragment(module: IrModuleFragment) { context.log("visitModule : ${ir2string(module)}") + prepareAllocHints(module, allocHints) + module.acceptChildrenVoid(this) appendLlvmUsed(context.llvm.usedFunctions) appendStaticInitializers(context.llvm.staticInitializers) @@ -691,10 +694,9 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid codegen.positionAtEnd(bbInit) val typeInfo = codegen.typeInfoValue(value.descriptor) - val allocHint = Int32(1).llvm val initFunction = value.descriptor.constructors.first { it.valueParameters.size == 0 } val ctor = codegen.llvmFunction(initFunction) - val args = listOf(objectPtr, typeInfo, allocHint, ctor) + val args = listOf(objectPtr, typeInfo, ctor) val newValue = call(context.llvm.initInstanceFunction, args) val bbInitResult = codegen.currentBlock codegen.br(bbExit) @@ -802,9 +804,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid codegen.plus(sum, size!!) } - val typeInfo = codegen.typeInfoValue(value.type)!! - val arrayCreationArgs = listOf(typeInfo, kImmInt32One, finalLength) - val array = call(context.llvm.allocArrayFunction, arrayCreationArgs) + val array = codegen.allocArray(codegen.typeInfoValue(value.type)!!, SCOPE_GLOBAL, finalLength) elements.fold(kImmZero) { sum, (exp, size, isArray) -> if (!isArray) { call(context.llvm.setArrayFunction, listOf(array, sum, exp)) @@ -832,6 +832,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val kStringLength = KonanPlatform.builtIns.string.getter2Descriptor(kNameLength) val kStringBuilderToString = kStringBuilder.signature2Descriptor(kNameToString) + //TODO: make it lowering pass. private fun evaluateStringConcatenation(value: IrStringConcatenation): LLVMValueRef { data class Element(val string: LLVMValueRef, val llvmLenght: LLVMValueRef?, val length: Int) @@ -857,8 +858,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val constructor = kStringBuilder!!.constructors .firstOrNull { it -> it.valueParameters.size == 1 && KotlinBuiltIns.isInt(it.valueParameters[0].type) }!! - val stringBuilderObj = call(context.llvm.allocInstanceFunction, - listOf(codegen.typeInfoValue(kStringBuilder.defaultType)!!, kImmOne)) + val stringBuilderObj = codegen.allocInstance(codegen.typeInfoValue(kStringBuilder), SCOPE_FRAME) + call(codegen.llvmFunction(constructor), listOf(stringBuilderObj, totalLength)) stringsWithLengths.fold(stringBuilderObj) { sum, (string, _, _) -> @@ -1434,11 +1435,10 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid */ private fun fieldPtrOfClass(thisPtr: LLVMValueRef, value: PropertyDescriptor): LLVMValueRef { - val objHeaderPtr = codegen.bitcast(codegen.kObjHeaderPtr, thisPtr) val typePtr = pointerType(codegen.classType(value.containingDeclaration as ClassDescriptor)) memScoped { val args = allocArrayOf(kImmOne) - val objectPtr = LLVMBuildGEP(codegen.builder, objHeaderPtr, args[0].ptr, 1, "") + val objectPtr = LLVMBuildGEP(codegen.builder, thisPtr, args[0].ptr, 1, "") val typedObjPtr = codegen.bitcast(typePtr, objectPtr!!) val fieldPtr = LLVMBuildStructGEP(codegen.builder, typedObjPtr, codegen.indexInClass(value), "") return fieldPtr!! @@ -1742,24 +1742,22 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid } //-------------------------------------------------------------------------// + private fun hintForCall(callee: IrCall): Int { + return allocHints.getOrElse(callee) { SCOPE_GLOBAL } + } private fun evaluateConstructorCall(callee: IrCall, args: List): LLVMValueRef { context.log("evaluateConstructorCall : ${ir2string(callee)}") memScoped { val constructedClass = (callee.descriptor as ConstructorDescriptor).constructedClass - val typeInfo = codegen.typeInfoValue(constructedClass) - val allocHint = Int32(1).llvm val thisValue = if (constructedClass.isArray) { assert(args.size >= 1 && args[0].type == int32Type) - val allocArrayInstanceArgs = listOf(typeInfo, allocHint, args[0]) - call(context.llvm.allocArrayFunction, allocArrayInstanceArgs) + codegen.allocArray(codegen.typeInfoValue(constructedClass), hintForCall(callee), args[0]) } else { - call(context.llvm.allocInstanceFunction, listOf(typeInfo, allocHint)) + codegen.allocInstance(codegen.typeInfoValue(constructedClass), hintForCall(callee)) } - val constructorParams: MutableList = mutableListOf() - constructorParams += thisValue - constructorParams += args - evaluateSimpleFunctionCall(callee.descriptor as FunctionDescriptor, constructorParams) + evaluateSimpleFunctionCall(callee.descriptor as FunctionDescriptor, + listOf(thisValue) + args) return thisValue } } @@ -1772,8 +1770,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid return when (name) { "konan.internal.areEqualByValue" -> { - val arg0 = args[0]!! - val arg1 = args[1]!! + val arg0 = args[0] + val arg1 = args[1] assert (arg0.type == arg1.type) when (LLVMGetTypeKind(arg0.type)) { @@ -1801,12 +1799,12 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val descriptor = callee.descriptor val ib = context.irModule!!.irBuiltins when (descriptor) { - ib.eqeqeq -> return codegen.icmpEq(args[0]!!, args[1]!!) - ib.gt0 -> return codegen.icmpGt(args[0]!!, kImmZero) - ib.gteq0 -> return codegen.icmpGe(args[0]!!, kImmZero) - ib.lt0 -> return codegen.icmpLt(args[0]!!, kImmZero) - ib.lteq0 -> return codegen.icmpLe(args[0]!!, kImmZero) - ib.booleanNot -> return codegen.icmpNe(args[0]!!, kTrue) + ib.eqeqeq -> return codegen.icmpEq(args[0], args[1]) + ib.gt0 -> return codegen.icmpGt(args[0], kImmZero) + ib.gteq0 -> return codegen.icmpGe(args[0], kImmZero) + ib.lt0 -> return codegen.icmpLt(args[0], kImmZero) + ib.lteq0 -> return codegen.icmpLe(args[0], kImmZero) + ib.booleanNot -> return codegen.icmpNe(args[0], kTrue) else -> { TODO(descriptor.name.toString()) } @@ -1838,7 +1836,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid if (resultPhi != null && !isNothing) codegen.assignPhis(resultPhi to brResult) if (bbExit != null && !isNothing) - codegen.br(bbExit!!) + codegen.br(bbExit) if (bbNext != null) // Switch generation to next or exit. codegen.positionAtEnd(bbNext) else if (bbExit != null) @@ -1878,7 +1876,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val slot = codegen.gep(vtable, Int32(index).llvm) codegen.load(slot) } else { - // Otherwise, call via hashtable. + // Otherwise, call by hash. // TODO: optimize by storing interface number in lower bits of 'this' pointer // when passing object as an interface. This way we can use those bits as index // for an additional per-interface vtable. @@ -1914,7 +1912,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid private fun call(function: LLVMValueRef, args: List): LLVMValueRef { if (codegen.isObjectReturn(function.type)) { // If function returns an object - create slot for the returned value. - // This allows appropriate rootset accounting by just looking on stack slots. + // This allows appropriate rootset accounting by just looking at the stack slots. val resultSlot = codegen.vars.createAnonymousSlot() return currentCodeContext.genCall(function, args + resultSlot) } else { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt index 6a8272a8a5d..ae5155e8330 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt @@ -115,6 +115,8 @@ internal val ContextUtils.kTypeInfo: LLVMTypeRef get() = LLVMGetTypeByName(context.llvmModule, "struct.TypeInfo")!! internal val ContextUtils.kObjHeader: LLVMTypeRef get() = LLVMGetTypeByName(context.llvmModule, "struct.ObjHeader")!! +internal val ContextUtils.kContainerHeader: LLVMTypeRef + get() = LLVMGetTypeByName(context.llvmModule, "struct.ContainerHeader")!! internal val ContextUtils.kObjHeaderPtr: LLVMTypeRef get() = pointerType(kObjHeader) internal val ContextUtils.kObjHeaderPtrPtr: LLVMTypeRef @@ -129,7 +131,7 @@ internal val kInt1 = LLVMInt1Type()!! internal val kBoolean = kInt1 internal val kInt8Ptr = pointerType(int8Type) internal val kInt8PtrPtr = pointerType(kInt8Ptr) -internal val kNullInt8Ptr = LLVMConstNull(kInt8Ptr) +internal val kNullInt8Ptr = LLVMConstNull(kInt8Ptr)!! internal val kImmInt32One = Int32(1).llvm internal val kImmInt64One = Int64(1).llvm internal val ContextUtils.kNullObjHeaderPtr: LLVMValueRef diff --git a/runtime/src/launcher/cpp/launcher.cpp b/runtime/src/launcher/cpp/launcher.cpp index 5e28b99c953..a077f1f266f 100644 --- a/runtime/src/launcher/cpp/launcher.cpp +++ b/runtime/src/launcher/cpp/launcher.cpp @@ -8,10 +8,10 @@ OBJ_GETTER(setupArgs, int argc, char** argv) { // The count is one less, because we skip argv[0] which is the binary name. - AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, argc - 1, OBJ_RESULT); + AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT); ArrayHeader* array = (*OBJ_RESULT)->array(); for (int index = 1; index < argc; index++) { - AllocStringInstance(SCOPE_GLOBAL, argv[index], strlen(argv[index]), + AllocStringInstance(argv[index], strlen(argv[index]), ArrayAddressOfElementAt(array, index - 1)); } RETURN_OBJ_RESULT(); diff --git a/runtime/src/main/cpp/Exceptions.cpp b/runtime/src/main/cpp/Exceptions.cpp index 6b704d67362..48755a5237d 100644 --- a/runtime/src/main/cpp/Exceptions.cpp +++ b/runtime/src/main/cpp/Exceptions.cpp @@ -50,13 +50,12 @@ OBJ_GETTER0(GetCurrentStackTrace) { RuntimeAssert(symbols != nullptr, "Not enough memory to retrieve the stacktrace"); AutoFree autoFree(symbols); - AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, size, OBJ_RESULT); + AllocArrayInstance(theArrayTypeInfo, size, OBJ_RESULT); ArrayHeader* array = (*OBJ_RESULT)->array(); for (int index = 0; index < size; ++index) { - AllocStringInstance( - SCOPE_GLOBAL, symbols[index], strlen(symbols[index]), - ArrayAddressOfElementAt(array, index)); + AllocStringInstance(symbols[index], strlen(symbols[index]), + ArrayAddressOfElementAt(array, index)); } RETURN_OBJ_RESULT(); diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 7c37af313fb..b5b3f2df7be 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -21,11 +21,16 @@ #define TRACE_GC_PHASES 0 ContainerHeader ObjHeader::theStaticObjectsContainer = { - CONTAINER_TAG_NOCOUNT | CONTAINER_TAG_INCREMENT + CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT }; namespace { +// Granularity of arena container chunks. +constexpr container_size_t kContainerAlignment = 1024; +// Single object alignment. +constexpr container_size_t kObjectAlignment = 8; + #if USE_GC // Collection threshold default (collect after having so many elements in the // release candidates set). @@ -63,11 +68,30 @@ struct MemoryState { MemoryState* memoryState = nullptr; -#if USE_GC -bool isPermanent(const ContainerHeader* header) { - return (header->ref_count_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NOCOUNT; +// TODO: use those allocators for STL containers as well. +template +inline T* allocMemory(container_size_t size) { + return reinterpret_cast(calloc(1, size)); } +inline void freeMemory(void* memory) { + free(memory); +} + +inline bool isFreeable(const ContainerHeader* header) { + return (header->refCount_ & CONTAINER_TAG_MASK) < CONTAINER_TAG_PERMANENT; +} + +inline bool isPermanent(const ContainerHeader* header) { + return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT; +} + +inline container_size_t alignUp(container_size_t size, int alignment) { + return (size + alignment - 1) & ~(alignment - 1); +} + +#if USE_GC + // Must be vector or map 'container -> number', to keep reference counters correct. ContainerHeaderList collectMutableReferred(ContainerHeader* header) { ContainerHeaderList result; @@ -98,8 +122,8 @@ ContainerHeaderList collectMutableReferred(ContainerHeader* header) { void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) { fprintf(stderr, "%s: %p (%08x): %d refs %s\n", prefix, - header, header->ref_count_, header->ref_count_ >> CONTAINER_TAG_SHIFT, - (header->ref_count_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-"); + header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT, + (header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-"); seen->insert(header); auto children = collectMutableReferred(header); for (auto child : children) { @@ -117,22 +141,22 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { } void phase1(ContainerHeader* header) { - if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0) + if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) return; - header->ref_count_ |= CONTAINER_TAG_SEEN; + header->refCount_ |= CONTAINER_TAG_SEEN; auto containers = collectMutableReferred(header); for (auto container : containers) { - container->ref_count_ -= CONTAINER_TAG_INCREMENT; + container->refCount_ -= CONTAINER_TAG_INCREMENT; phase1(container); } } void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) { - if ((header->ref_count_ & CONTAINER_TAG_SEEN) == 0) + if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0) return; - if ((header->ref_count_ >> CONTAINER_TAG_SHIFT) != 0) + if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0) rootset->insert(header); - header->ref_count_ &= ~CONTAINER_TAG_SEEN; + header->refCount_ &= ~CONTAINER_TAG_SEEN; auto containers = collectMutableReferred(header); for (auto container : containers) { phase2(container, rootset); @@ -140,32 +164,32 @@ void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) { } void phase3(ContainerHeader* header) { - if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0) { + if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) { return; } - header->ref_count_ |= CONTAINER_TAG_SEEN; + header->refCount_ |= CONTAINER_TAG_SEEN; auto containers = collectMutableReferred(header); for (auto container : containers) { - container->ref_count_ += CONTAINER_TAG_INCREMENT; + container->refCount_ += CONTAINER_TAG_INCREMENT; phase3(container); } } void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) { - auto ref_count = header->ref_count_ >> CONTAINER_TAG_SHIFT; - bool seen = (ref_count > 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) == 0) || - (ref_count == 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) != 0); + auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT; + bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) || + (refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0); if (seen) return; // Add to toRemove set. - if (ref_count == 0) + if (refCount == 0) toRemove->insert(header); // Update seen bit. - if (ref_count == 0) - header->ref_count_ |= CONTAINER_TAG_SEEN; + if (refCount == 0) + header->refCount_ |= CONTAINER_TAG_SEEN; else - header->ref_count_ &= ~CONTAINER_TAG_SEEN; + header->refCount_ &= ~CONTAINER_TAG_SEEN; auto containers = collectMutableReferred(header); for (auto container : containers) { phase4(container, toRemove); @@ -174,91 +198,109 @@ void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) { #endif // USE_GC +// We use first slot as place to store frame-local arena container. +ArenaContainer* initedArena(ObjHeader** auxSlot) { + ObjHeader* slotValue = *auxSlot; + if (slotValue) return reinterpret_cast(slotValue); + ArenaContainer* arena = allocMemory(sizeof(ArenaContainer)); + arena->Init(); + *auxSlot = reinterpret_cast(arena); + return arena; +} + } // namespace ContainerHeader* AllocContainer(size_t size) { - ContainerHeader* result = reinterpret_cast(calloc(1, size)); + ContainerHeader* result = allocMemory(size); #if TRACE_MEMORY - fprintf(stderr, ">>> alloc %d -> %p\n", (int)size, result); - memoryState->containers->insert(result); + fprintf(stderr, ">>> alloc %d -> %p\n", static_cast(size), result); + memoryState->containers->insert(result); #endif // TODO: atomic increment in concurrent case. memoryState->allocCount++; return result; } +// TODO: shall we do padding for alignment? +uint32_t ObjectSize(const ObjHeader* obj) { + const TypeInfo* type_info = obj->type_info(); + if (type_info->instanceSize_ < 0) { + // An array. + return ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader); + } else { + return type_info->instanceSize_ + sizeof(ObjHeader); + } +} + void FreeContainer(ContainerHeader* header) { RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed"); #if TRACE_MEMORY - fprintf(stderr, "<<< free %p\n", header); - memoryState->containers->erase(header); + if (isFreeable(header)) { + fprintf(stderr, "<<< free %p\n", header); + memoryState->containers->erase(header); + } #endif - header->ref_count_ = CONTAINER_TAG_INVALID; + #if USE_GC - if (memoryState->toFree) + if (memoryState->toFree && isFreeable(header)) memoryState->toFree->erase(header); #endif // Now let's clean all object's fields in this container. - // TODO: this is gross hack, relying on the fact that we now only alloc - // ArenaContainer and ObjectContainer, which both have single element. ObjHeader* obj = reinterpret_cast(header + 1); - const TypeInfo* typeInfo = obj->type_info(); - // We use *local* versions as no other threads could see dead objects. - for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { - ObjHeader** location = reinterpret_cast( - reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); - UpdateLocalRef(location, nullptr); - } - // Object arrays are *special*. - if (typeInfo == theArrayTypeInfo) { - ArrayHeader* array = obj->array(); - ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_); + for (int index = 0; index < header->objectCount_; index++) { + const TypeInfo* typeInfo = obj->type_info(); + + // We use *local* versions as no other threads could see dead objects. + for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { + ObjHeader** location = reinterpret_cast( + reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); + UpdateLocalRef(location, nullptr); + } + // Object arrays are *special*. + if (typeInfo == theArrayTypeInfo) { + ArrayHeader* array = obj->array(); + ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_); + } + obj = reinterpret_cast(reinterpret_cast(obj) + ObjectSize(obj)); } // And release underlying memory. - // TODO: atomic decrement in concurrent case. + if (isFreeable(header)) { + // TODO: atomic decrement in concurrent case. #if CONCURRENT - #error "Atomic update of allocCount" + #error "Atomic update of allocCount" #endif - memoryState->allocCount--; - free(header); + memoryState->allocCount--; + freeMemory(header); + } } #if USE_GC void FreeContainerNoRef(ContainerHeader* header) { - RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed"); + RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed"); #if TRACE_MEMORY fprintf(stderr, "<<< free %p\n", header); memoryState->containers->erase(header); #endif - header->ref_count_ = CONTAINER_TAG_INVALID; #if USE_GC if (memoryState->toFree) memoryState->toFree->erase(header); #endif memoryState->allocCount--; - free(header); + freeMemory(header); } #endif -ArenaContainer::ArenaContainer(uint32_t size) { - ArenaContainerHeader* header = - static_cast(AllocContainer(size + sizeof(ArenaContainerHeader))); - header_ = header; - // header->ref_count_ is zero initialized by AllocContainer(). - header->current_ = - reinterpret_cast(header_) + sizeof(ArenaContainerHeader); - header->end_ = header->current_ + size; -} - void ObjectContainer::Init(const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object"); uint32_t alloc_size = sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_; header_ = AllocContainer(alloc_size); if (header_) { - // header->ref_count_ is zero initialized by AllocContainer(). + // One object in this container. + header_->objectCount_ = 1; + // header->refCount_ is zero initialized by AllocContainer(). SetMeta(GetPlace(), type_info); #if TRACE_MEMORY fprintf(stderr, "object at %p\n", GetPlace()); @@ -274,7 +316,9 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) { header_ = AllocContainer(alloc_size); RuntimeAssert(header_ != nullptr, "Cannot alloc memory"); if (header_) { - // header->ref_count_ is zero initialized by AllocContainer(). + // One object in this container. + header_->objectCount_ = 1; + // header->refCount_ is zero initialized by AllocContainer(). GetPlace()->count_ = elements; SetMeta(GetPlace()->obj(), type_info); #if TRACE_MEMORY @@ -283,25 +327,73 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) { } } -ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { - RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); - uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader); - ObjHeader* result = reinterpret_cast(Place(size)); - if (!result) { - return nullptr; +void ArenaContainer::Init() { + allocContainer(1024); +} + +void ArenaContainer::Deinit() { + auto chunk = currentChunk_; + while (chunk != nullptr) { + auto toRemove = chunk; + // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set. + FreeContainer(chunk->asHeader()); + chunk = chunk->next; + freeMemory(toRemove); } - SetMeta(result, type_info); +} + +bool ArenaContainer::allocContainer(container_size_t minSize) { + auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk); + size = alignUp(size, kContainerAlignment); + ContainerChunk* result = allocMemory(size); + RuntimeAssert(result != nullptr, "Cannot alloc memory"); + if (result == nullptr) return false; + result->next = currentChunk_; + result->asHeader()->refCount_ = (CONTAINER_TAG_STACK | CONTAINER_TAG_INCREMENT); + currentChunk_ = result; + current_ = reinterpret_cast(result->asHeader() + 1); + end_ = reinterpret_cast(result) + size; + return true; +} + +void* ArenaContainer::place(container_size_t size) { + size = alignUp(size, kObjectAlignment); + // Fast path. + if (current_ + size < end_) { + void* result = current_; + current_ += size; + return result; + } + if (!allocContainer(size)) { + return nullptr; + } + void* result = current_; + current_ += size; + RuntimeAssert(current_ <= end_, "Must not overflow"); return result; } -ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, int count) { +ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { + RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); + uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader); + ObjHeader* result = reinterpret_cast(place(size)); + if (!result) { + return nullptr; + } + currentChunk_->asHeader()->objectCount_++; + setMeta(result, type_info); + return result; +} + +ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t count) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); - uint32_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count; - ArrayHeader* result = reinterpret_cast(Place(size)); + container_size_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count; + ArrayHeader* result = reinterpret_cast(place(size)); if (!result) { return nullptr; } - SetMeta(result->obj(), type_info); + currentChunk_->asHeader()->objectCount_++; + setMeta(result->obj(), type_info); result->count_ = count; return result; } @@ -384,6 +476,8 @@ void DeinitMemory() { #if USE_GC GarbageCollect(); + delete memoryState->toFree; + memoryState->toFree = nullptr; #endif // USE_GC if (memoryState->allocCount > 0) { @@ -399,20 +493,28 @@ void DeinitMemory() { memoryState = nullptr; } -// Now we ignore all placement hints and always allocate heap space for new object. -OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint) { +ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) { + RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); + return initedArena(auxSlot)->PlaceObject(type_info); +} + +OBJ_GETTER(AllocInstance, const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object"); RETURN_OBJ(ObjectContainer(type_info).GetPlace()); } -OBJ_GETTER(AllocArrayInstance, - const TypeInfo* type_info, PlacementHint hint, uint32_t elements) { +ObjHeader* ArenaAllocArrayInstance( + const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) { + RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); + return initedArena(auxSlot)->PlaceArray(type_info, elements)->obj(); +} + +OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) { RuntimeAssert(type_info->instanceSize_ < 0, "must be an array"); RETURN_OBJ(ArrayContainer(type_info, elements).GetPlace()->obj()); } -OBJ_GETTER(AllocStringInstance, - PlacementHint hint, const char* data, uint32_t length) { +OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) { ArrayHeader* array = ArrayContainer(theStringTypeInfo, length).GetPlace(); memcpy( ByteArrayAddressOfElementAt(array, 0), @@ -422,8 +524,7 @@ OBJ_GETTER(AllocStringInstance, } OBJ_GETTER(InitInstance, - ObjHeader** location, const TypeInfo* type_info, PlacementHint hint, - void (*ctor)(ObjHeader*)) { + ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) { ObjHeader* sentinel = reinterpret_cast(1); ObjHeader* value; // Wait until other initializers. @@ -438,7 +539,7 @@ OBJ_GETTER(InitInstance, RETURN_OBJ(value); } - AllocInstance(type_info, hint, OBJ_RESULT); + AllocInstance(type_info, OBJ_RESULT); ObjHeader* object = *OBJ_RESULT; UpdateGlobalRef(location, object); try { @@ -523,6 +624,15 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) { #endif } +void LeaveFrame(ObjHeader** start, int count) { + ReleaseLocalRefs(start + 1, count - 1); + if (*start != nullptr) { + auto arena = initedArena(start); + arena->Deinit(); + freeMemory(arena); + } +} + void ReleaseLocalRefs(ObjHeader** start, int count) { #if TRACE_MEMORY fprintf(stderr, "ReleaseLocalRefs %p .. %p\n", start, start + count); @@ -623,7 +733,7 @@ void GarbageCollect() { memoryState->toFree->clear(); for (auto header : toRemove) { - RuntimeAssert((header->ref_count_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen"); + RuntimeAssert((header->refCount_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen"); FreeContainerNoRef(header); } diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 6748e893e2d..7db58109722 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -10,7 +10,7 @@ typedef enum { SCOPE_FRAME = 0, // Allocation is generic global allocation. SCOPE_GLOBAL = 1, - // Allocation shall take place in current arena. + // Allocation shall take place in current stack arena. SCOPE_ARENA = 2, // Allocation is permanent. SCOPE_PERMANENT = 3 @@ -20,15 +20,16 @@ typedef enum { typedef enum { // Container is normal thread local container. CONTAINER_TAG_NORMAL = 0, - // Container shall not be refcounted (const data, frame locals). - CONTAINER_TAG_NOCOUNT = 1, - // Container shall be atomically refcounted. - CONTAINER_TAG_SHARED = 2, - // Container is no longer valid. - CONTAINER_TAG_INVALID = 3, + // Container shall be atomically refcounted. + CONTAINER_TAG_SHARED = 1, + // Those container tags shall not be refcounted. + // Permanent object, cannot refer to non-permanent objects, so no need to cleanup those. + CONTAINER_TAG_PERMANENT = 2, + // Stack objects, no need to free, children cleanup still shall be there. + CONTAINER_TAG_STACK = 3, // Container was seen during GC. CONTAINER_TAG_SEEN = 4, - // Shift to get actual counter.. + // Shift to get actual counter. CONTAINER_TAG_SHIFT = 3, // Actual value to increment/decrement conatiner by. Tag is in lower bits. CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT, @@ -36,14 +37,17 @@ typedef enum { CONTAINER_TAG_MASK = ((CONTAINER_TAG_INCREMENT >> 1) - 1) } ContainerTag; -// Could be made 64-bit for large memory configs. typedef uint32_t container_offset_t; +typedef uint32_t container_size_t; + // Header of all container objects. Contains reference counter. struct ContainerHeader { // Reference counter of container. Uses two lower bits of counter for // container type (for polymorphism in ::Release()). - volatile uint32_t ref_count_; + volatile uint32_t refCount_; + // Number of objects in the container. + uint32_t objectCount_; }; struct ArrayHeader; @@ -116,37 +120,25 @@ struct ArrayHeader { uint32_t count_; }; -struct ArenaContainerHeader : public ContainerHeader { - // Current allocation limit. - uint8_t* current_; - // Allocation end. Maybe consider having chunked backing storage - // at cost of smarter ::Release() polymorphic on container type. - uint8_t* end_; -}; - inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { // Instance size is negative. return -obj->type_info()->instanceSize_ * obj->count_; } -void FreeContainer(ContainerHeader* header); - // TODO: those two operations can be implemented by translator when storing // reference to an object. inline void AddRef(ContainerHeader* header) { // Looking at container type we may want to skip AddRef() totally // (non-escaping stack objects, constant objects). - switch (header->ref_count_ & CONTAINER_TAG_MASK) { - case CONTAINER_TAG_NORMAL: - header->ref_count_ += CONTAINER_TAG_INCREMENT; + switch (header->refCount_ & CONTAINER_TAG_MASK) { + case CONTAINER_TAG_STACK: + case CONTAINER_TAG_PERMANENT: break; - case CONTAINER_TAG_NOCOUNT: + case CONTAINER_TAG_NORMAL: + header->refCount_ += CONTAINER_TAG_INCREMENT; break; case CONTAINER_TAG_SHARED: - __sync_fetch_and_add(&header->ref_count_, CONTAINER_TAG_INCREMENT); - break; - case CONTAINER_TAG_INVALID: - RuntimeAssert(false, "trying to addref invalid container"); + __sync_fetch_and_add(&header->refCount_, CONTAINER_TAG_INCREMENT); break; default: RuntimeAssert(false, "unknown container type"); @@ -154,19 +146,22 @@ inline void AddRef(ContainerHeader* header) { } } -// Release returns 'true' iff container cannot be part of cycle (either NOCOUNT +void FreeContainer(ContainerHeader* header); + +// Release() returns 'true' iff container cannot be part of cycle (either NOCOUNT // object or container was fully released and will be collected). inline bool Release(ContainerHeader* header) { - switch (header->ref_count_ & CONTAINER_TAG_MASK) { + switch (header->refCount_ & CONTAINER_TAG_MASK) { + case CONTAINER_TAG_PERMANENT: + case CONTAINER_TAG_STACK: + // permanent/stack containers aren't loop candidates. + return true; case CONTAINER_TAG_NORMAL: - if ((header->ref_count_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) { + if ((header->refCount_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) { FreeContainer(header); return true; } break; - case CONTAINER_TAG_NOCOUNT: - // NOCOUNT containers aren't loop candidate. - return true; // Note that shared containers have potentially subtle race, if object holds a // reference to another object, stored in shorter living container. In this // case there's unlikely, but possible case, where one mutator takes reference, @@ -182,14 +177,11 @@ inline bool Release(ContainerHeader* header) { // probability even further. case CONTAINER_TAG_SHARED: if (__sync_sub_and_fetch( - &header->ref_count_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) { + &header->refCount_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) { FreeContainer(header); return true; } break; - case CONTAINER_TAG_INVALID: - RuntimeAssert(false, "trying to release invalid container"); - break; default: RuntimeAssert(false, "unknown container type"); break; @@ -236,9 +228,9 @@ class ObjectContainer : public Container { // Object container shalln't have any dtor, as it's being freed by // ::Release(). + ObjHeader* GetPlace() const { - return reinterpret_cast( - reinterpret_cast(header_) + sizeof(ContainerHeader)); + return reinterpret_cast(header_ + 1); } private: @@ -253,41 +245,23 @@ class ArrayContainer : public Container { } // Array container shalln't have any dtor, as it's being freed by ::Release(). + ArrayHeader* GetPlace() const { - return reinterpret_cast( - reinterpret_cast(header_) + sizeof(ContainerHeader)); + return reinterpret_cast(header_ + 1); } private: void Init(const TypeInfo* type_info, uint32_t elements); }; - // Class representing arena-style placement container. -// Container is used for reference counting, -// and it is assumed that objects with related placement will share container. Only +// Container is used for reference counting, and it is assumed that objects +// with related placement will share container. Only // whole container can be freed, individual objects are not taken into account. -class ArenaContainer : public Container { +class ArenaContainer { public: - explicit ArenaContainer(uint32_t size); - - ~ArenaContainer() { - if (header_) { - RuntimeAssert(header_->ref_count_ == 0, "Non-zero refcount"); - Dispose(); - } - } - - // Allocation function. - void* Place(int size) { - ArenaContainerHeader* header = reinterpret_cast(header_); - if (header->current_ + size > header->end_) { - return nullptr; - } - void* result = header->current_; - header->current_ += size; - return result; - } + void Init(); + void Deinit(); // Place individual object in this container. ObjHeader* PlaceObject(const TypeInfo* type_info); @@ -295,15 +269,28 @@ class ArenaContainer : public Container { // Places an array of certain type in this container. Note that array_type_info // is type info for an array, not for an individual element. Also note that exactly // same operation could be used to place strings. - ArrayHeader* PlaceArray(const TypeInfo* array_type_info, int count); + ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count); - // Dispose whole container ignoring non-zero refcount. Use with care. - void Dispose() { - if (header_) { - FreeContainer(header_); - header_ = nullptr; + private: + struct ContainerChunk { + ContainerChunk* next; + // Then we have ContainerHeader here. + ContainerHeader* asHeader() { + return reinterpret_cast(this + 1); } + }; + + void* place(container_size_t size); + bool allocContainer(container_size_t minSize); + void setMeta(ObjHeader* obj, const TypeInfo* type_info) { + obj->container_offset_negative_ = + reinterpret_cast(obj) - reinterpret_cast(currentChunk_->asHeader()); + obj->set_type_info(type_info); + RuntimeAssert(obj->container() == currentChunk_->asHeader(), "Placement must match"); } + ContainerChunk* currentChunk_; + uint8_t* current_; + uint8_t* end_; }; #ifdef __cplusplus @@ -321,14 +308,26 @@ extern "C" { void InitMemory(); void DeinitMemory(); -OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint); -OBJ_GETTER(AllocArrayInstance, - const TypeInfo* type_info, PlacementHint hint, uint32_t elements); -OBJ_GETTER(AllocStringInstance, - PlacementHint hint, const char* data, uint32_t length); +// +// Object allocation. +// +// Allocation can happen in either GLOBAL, FRAME or ARENA scope. Depending on that, +// Alloc* or ArenaAlloc* is called. Regular alloc means allocation happens in the heap, +// and each object gets its individual container. Otherwise, allocator uses aux slot in +// an implementation-defined manner, current behavior is to keep arena pointer there. +// Arena containers are not reference counted, and is explicitly freed when leaving +// its owner frame. +// Escape analysis algorithm is the provider of information for decision on exact aux slot +// selection, and comes from upper bound esteemation of object lifetime. +// +ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) RUNTIME_NOTHROW; +OBJ_GETTER(AllocInstance, const TypeInfo* type_info) RUNTIME_NOTHROW; +ObjHeader* ArenaAllocArrayInstance( + const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) RUNTIME_NOTHROW; +OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) RUNTIME_NOTHROW; +OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) RUNTIME_NOTHROW; OBJ_GETTER(InitInstance, - ObjHeader** location, const TypeInfo* type_info, PlacementHint hint, - void (*ctor)(ObjHeader*)); + ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) RUNTIME_NOTHROW; // // Object reference management. @@ -363,6 +362,8 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTH // Optimization: release all references in range. void ReleaseLocalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; void ReleaseGlobalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; +// Called on frame leave, if it has object slots. +void LeaveFrame(ObjHeader** start, int count) RUNTIME_NOTHROW; // Collect garbage, which cannot be found by reference counting (cycles). void GarbageCollect() RUNTIME_NOTHROW; diff --git a/runtime/src/main/cpp/Operator.cpp b/runtime/src/main/cpp/Operator.cpp index 0408aa043f3..d24244a8466 100644 --- a/runtime/src/main/cpp/Operator.cpp +++ b/runtime/src/main/cpp/Operator.cpp @@ -5,7 +5,7 @@ namespace { -inline template R div(Ta a, Tb b) { +template R div(Ta a, Tb b) { if (__builtin_expect(b == 0, false)) { ThrowArithmeticException(); } diff --git a/runtime/src/main/kotlin/kotlin/Arrays.kt b/runtime/src/main/kotlin/kotlin/Arrays.kt index 182585a5b92..b5fa8b5985a 100644 --- a/runtime/src/main/kotlin/kotlin/Arrays.kt +++ b/runtime/src/main/kotlin/kotlin/Arrays.kt @@ -2,6 +2,8 @@ package kotlin import kotlin.collections.* +// TODO: make all iterator() methods inline. + /** * An array of bytes. * @constructor Creates a new array of the specified [size], with all elements initialized to zero. @@ -27,7 +29,7 @@ public final class ByteArray : Cloneable { external private fun getArrayLength(): Int /** Creates an iterator over the elements of the array. */ - public operator fun iterator(): kotlin.collections.ByteIterator { + public operator fun iterator(): ByteIterator { return ByteIteratorImpl(this) } }