From e65b86ab2115d58bc6ef20263d0360a5339cf6fd Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Tue, 27 Dec 2016 15:01:18 +0200 Subject: [PATCH] Memory management design and implementation. (#148) --- .../backend/konan/llvm/CodeGenerator.kt | 33 ++- .../kotlin/backend/konan/llvm/ContextUtils.kt | 2 + .../kotlin/backend/konan/llvm/IrToBitcode.kt | 10 +- .../kotlin/backend/konan/llvm/LlvmUtils.kt | 13 +- .../backend/konan/llvm/RTTIGenerator.kt | 2 +- .../backend/konan/llvm/VariableManager.kt | 11 +- .../tests/runtime/collections/array_list1.kt | 4 +- backend.native/tests/runtime/memory/basic0.kt | 10 + runtime/src/launcher/cpp/launcher.cpp | 12 +- runtime/src/main/cpp/Arrays.cpp | 31 +-- runtime/src/main/cpp/Memory.cpp | 203 ++++++++++++++++-- runtime/src/main/cpp/Memory.h | 53 ++++- runtime/src/main/cpp/Types.cpp | 5 + runtime/src/main/cpp/Types.h | 1 + 14 files changed, 329 insertions(+), 61 deletions(-) create mode 100644 backend.native/tests/runtime/memory/basic0.kt diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 043c2f14201..dfed5a73c23 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -103,20 +103,47 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun intToPtr(imm: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, imm, DestTy, Name) - fun alloca(type: KotlinType, name: String = ""): LLVMValueRef = alloca(getLLVMType(type), name) fun alloca(type: LLVMTypeRef?, name: String = ""): LLVMValueRef { appendingTo(prologueBb!!) { - return LLVMBuildAlloca(builder, type, name)!! + val result = LLVMBuildAlloca(builder, type, name)!! + if (isObjectType(type!!)) + LLVMBuildStore(builder, kNullObjHeaderPtr, result) + return result } } fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildLoad(builder, value, name)!! - fun store(value: LLVMValueRef, ptr: LLVMValueRef): LLVMValueRef = LLVMBuildStore(builder, value, ptr)!! + fun store(value: LLVMValueRef, ptr: LLVMValueRef) { + // Use updateRef() or storeAny() API for that. + assert(!isObjectRef(value)) + LLVMBuildStore(builder, value, ptr) + } + fun storeAnyLocal(value: LLVMValueRef, ptr: LLVMValueRef) { + if (isObjectRef(value)) { + updateLocalRef(value, ptr) + } else { + LLVMBuildStore(builder, value, ptr) + } + } + fun storeAnyGlobal(value: LLVMValueRef, ptr: LLVMValueRef) { + if (isObjectRef(value)) { + updateGlobalRef(value, ptr) + } else { + LLVMBuildStore(builder, value, ptr) + } + } + + // Only use ignoreOld, when sure that memory is freshly inited and have no value. fun updateLocalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) { call(if (ignoreOld) context.llvm.setLocalRefFunction else context.llvm.updateLocalRefFunction, listOf(address, value)) } + fun updateGlobalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) { + call(if (ignoreOld) context.llvm.setGlobalRefFunction else context.llvm.updateGlobalRefFunction, + listOf(address, value)) + } + fun isConst(value: LLVMValueRef): Boolean = (LLVMIsConstant(value) == 1) //-------------------------------------------------------------------------// diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 9db8b4440b6..fdcc021bd1f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -222,7 +222,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val initInstanceFunction = importRtFunction("InitInstance") val allocArrayFunction = importRtFunction("AllocArrayInstance") val setLocalRefFunction = importRtFunction("SetLocalRef") + val setGlobalRefFunction = importRtFunction("SetGlobalRef") val updateLocalRefFunction = importRtFunction("UpdateLocalRef") + val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef") val setArrayFunction = importRtFunction("Kotlin_Array_set") val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl") val lookupFieldOffset = importRtFunction("LookupFieldOffset") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt index 77f1381c9ce..68e233f78a7 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt @@ -204,7 +204,6 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid * Convenient [InnerScope] implementation that is bound to the [currentCodeContext]. */ private abstract inner class InnerScopeImpl : InnerScope(currentCodeContext) - /** * Executes [block] with [codeContext] substituted as [currentCodeContext]. */ @@ -250,7 +249,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val descriptor = irField.descriptor val initialization = evaluateExpression(irField.initializer) val globalPtr = LLVMGetNamedGlobal(context.llvmModule, descriptor.symbolName) - codegen.store(initialization!!, globalPtr!!) + codegen.storeAnyGlobal(initialization!!, globalPtr!!) } codegen.ret(null) } @@ -412,7 +411,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid fieldDeclaration.initializer?.let { val value = evaluateExpression(it)!! val fieldPtr = fieldPtrOfClass(thisPtr, fieldDescriptor) - codegen.store(value, fieldPtr) + codegen.storeAnyGlobal(value, fieldPtr) } } @@ -1450,11 +1449,11 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid val valueToAssign = evaluateExpression(value.value)!! if (value.descriptor.dispatchReceiverParameter != null) { val thisPtr = instanceFieldAccessReceiver(value) - codegen.store(valueToAssign, fieldPtrOfClass(thisPtr, value.descriptor)) + codegen.storeAnyGlobal(valueToAssign, fieldPtrOfClass(thisPtr, value.descriptor)) } else { val globalValue = LLVMGetNamedGlobal(context.llvmModule, value.descriptor.symbolName) - codegen.store(valueToAssign, globalValue!!) + codegen.storeAnyGlobal(valueToAssign, globalValue!!) } return null @@ -1818,6 +1817,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid branch: IrBranch, bbNext: LLVMBasicBlockRef?, bbExit: LLVMBasicBlockRef?) { val neitherUnitNorNothing = !isNothing && !isUnit // If branches doesn't end with 'return' either result hasn't got 'unit' type. val branchResult = branch.result + // TODO: use phis here! if (isUnconditional(branch)) { // It is the "else" clause. val brResult = evaluateExpression(branchResult) // Generate clause body. if (neitherUnitNorNothing) // If nor unit neither result ends with return diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt index da72c7b1e9c..06e6cfc76c1 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt @@ -142,10 +142,6 @@ internal fun structType(types: List): LLVMTypeRef = memScoped { LLVMStructType(allocArrayOf(types)[0].ptr, types.size, 0)!! } -internal fun ContextUtils.isObjectType(type: LLVMTypeRef) : Boolean { - return type == kObjHeaderPtr || type == kArrayHeaderPtr -} - internal fun ContextUtils.getLlvmFunctionType(function: FunctionDescriptor): LLVMTypeRef { val returnType = if (function is ConstructorDescriptor) voidType else getLLVMType(function.returnType!!) val paramTypes = ArrayList(function.allValueParameters.map { getLLVMType(it.type) }) @@ -168,6 +164,14 @@ internal fun ContextUtils.isObjectReturn(functionType: LLVMTypeRef) : Boolean { return isObjectType(returnType) } +internal fun ContextUtils.isObjectRef(value: LLVMValueRef): Boolean { + return isObjectType(value.type) +} + +internal fun ContextUtils.isObjectType(type: LLVMTypeRef): Boolean { + return type == kObjHeaderPtr || type == kArrayHeaderPtr +} + /** * Reads [size] bytes contained in this array. */ @@ -208,6 +212,7 @@ internal fun functionType(returnType: LLVMTypeRef, isVarArg: Boolean = false, va LLVMFunctionType(returnType, paramTypesPtr, paramTypes.size, if (isVarArg) 1 else 0)!! } + fun llvm2string(value: LLVMValueRef?): String { if (value == null) return "" return LLVMPrintValueToString(value)!!.asCString().toString() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt index 5c6a91b34fd..4bbe52711dc 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt @@ -160,7 +160,7 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils { val refFieldIndices = classDesc.fields.mapIndexedNotNull { index, field -> val type = field.returnType!! - if (!KotlinBuiltIns.isPrimitiveType(type)) { + if (isObjectType(getLLVMType(type))) { index } else { null diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt index e2891a26d7e..88a4586568f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt @@ -18,7 +18,7 @@ internal class VariableManager(val codegen: CodeGenerator) { return codegen.load(address) } override fun store(value: LLVMValueRef) { - codegen.store(value, address) + codegen.storeAnyLocal(value, address) } override fun address() : LLVMValueRef { return this.address @@ -54,10 +54,13 @@ internal class VariableManager(val codegen: CodeGenerator) { fun releaseVars() { // This function is called by codegen to cleanup local references when leaving frame. + for (variable in variables) { + if (variable.isRefSlot()) + codegen.updateLocalRef(codegen.kNullObjHeaderPtr, variable.address()) + } } fun createVariable(scoped: Pair, value: LLVMValueRef? = null) : Int { - // TODO: fix, due to the bug in frontend, we shall always create stack slot for variable now. // Note that we always create slot for object references for memory management. val descriptor = scoped.first if (descriptor.isVar() || codegen.isObjectType(codegen.getLLVMType(descriptor.type)) || true) { @@ -74,7 +77,7 @@ internal class VariableManager(val codegen: CodeGenerator) { val type = codegen.getLLVMType(descriptor.type) val slot = codegen.alloca(type, descriptor.name.asString()) if (value != null) - codegen.store(value, slot) + codegen.storeAnyLocal(value, slot) variables.add(SlotRecord(slot, codegen.isObjectType(type))) descriptors[scoped] = index return index @@ -95,7 +98,7 @@ internal class VariableManager(val codegen: CodeGenerator) { val index = variables.size val slot = codegen.alloca(type) if (value != null) - codegen.store(value, slot) + codegen.storeAnyLocal(value, slot) variables.add(SlotRecord(slot, codegen.isObjectType(type))) return index } diff --git a/backend.native/tests/runtime/collections/array_list1.kt b/backend.native/tests/runtime/collections/array_list1.kt index 11717cff265..945d756c621 100644 --- a/backend.native/tests/runtime/collections/array_list1.kt +++ b/backend.native/tests/runtime/collections/array_list1.kt @@ -10,12 +10,12 @@ fun assertFalse(cond: Boolean) { fun assertEquals(value1: Any?, value2: Any?) { if (value1 != value2) - println("FAIL") + throw Error("FAIL " + value1 + " " + value2) } fun assertEquals(value1: Int, value2: Int) { if (value1 != value2) - println("FAIL") + throw Error("FAIL " + value1 + " " + value2) } fun testBasic() { diff --git a/backend.native/tests/runtime/memory/basic0.kt b/backend.native/tests/runtime/memory/basic0.kt new file mode 100644 index 00000000000..d8ed310c35f --- /dev/null +++ b/backend.native/tests/runtime/memory/basic0.kt @@ -0,0 +1,10 @@ +class A { + var field: B? = null +} + +class B(var field: Int) + +fun main(args : Array) { + val a = A() + a.field = B(2) +} diff --git a/runtime/src/launcher/cpp/launcher.cpp b/runtime/src/launcher/cpp/launcher.cpp index 5cb8451e5ba..5e28b99c953 100644 --- a/runtime/src/launcher/cpp/launcher.cpp +++ b/runtime/src/launcher/cpp/launcher.cpp @@ -1,4 +1,5 @@ #include + #include "Memory.h" #include "Natives.h" #include "Types.h" @@ -9,16 +10,15 @@ OBJ_GETTER(setupArgs, int argc, char** argv) { // The count is one less, because we skip argv[0] which is the binary name. AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, argc - 1, OBJ_RESULT); ArrayHeader* array = (*OBJ_RESULT)->array(); - for (int index = 0; index < argc - 1; index++) { - AllocStringInstance(SCOPE_GLOBAL, argv[index + 1], strlen(argv[index + 1]), - ArrayAddressOfElementAt(array, index)); + for (int index = 1; index < argc; index++) { + AllocStringInstance(SCOPE_GLOBAL, argv[index], strlen(argv[index]), + ArrayAddressOfElementAt(array, index - 1)); } RETURN_OBJ_RESULT(); } //--- main --------------------------------------------------------------------// - -extern "C" void Konan_start(ObjHeader* ); +extern "C" void Konan_start(const ObjHeader* ); int main(int argc, char** argv) { @@ -31,6 +31,8 @@ int main(int argc, char** argv) { Konan_start(args.obj()); } + DeinitMemory(); + // Yes, we have to follow Java convention and return zero. return 0; } diff --git a/runtime/src/main/cpp/Arrays.cpp b/runtime/src/main/cpp/Arrays.cpp index 8c9fa0a343e..93ed72e297f 100644 --- a/runtime/src/main/cpp/Arrays.cpp +++ b/runtime/src/main/cpp/Arrays.cpp @@ -32,10 +32,10 @@ OBJ_GETTER(Kotlin_Array_clone, KConstRef thiz) { const ArrayHeader* array = thiz->array(); ArrayHeader* result = ArrayContainer( array->type_info(), array->count_).GetPlace(); - memcpy( - ArrayAddressOfElementAt(result, 0), - ArrayAddressOfElementAt(array, 0), - ArrayDataSizeBytes(array)); + for (int index = 0; index < array->count_; index++) { + SetGlobalRef( + ArrayAddressOfElementAt(result, index), *ArrayAddressOfElementAt(array, index)); + } RETURN_OBJ(result->obj()); } @@ -49,9 +49,8 @@ void Kotlin_Array_fillImpl(KRef thiz, KInt fromIndex, KInt toIndex, KRef value) if (fromIndex < 0 || toIndex < fromIndex || toIndex > array->count_) { ThrowArrayIndexOutOfBoundsException(); } - // TODO: refcounting! for (KInt index = fromIndex; index < toIndex; ++index) { - *ArrayAddressOfElementAt(array, index) = value; + UpdateGlobalRef(ArrayAddressOfElementAt(array, index), value); } } @@ -59,13 +58,21 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex, KRef destination, KInt toIndex, KInt count) { const ArrayHeader* array = thiz->array(); ArrayHeader* destinationArray = destination->array(); - if (fromIndex < 0 || fromIndex + count > array->count_ || - toIndex < 0 || toIndex + count > destinationArray->count_) { - ThrowArrayIndexOutOfBoundsException(); + if (fromIndex < 0 || fromIndex + count > array->count_ || + toIndex < 0 || toIndex + count > destinationArray->count_) { + ThrowArrayIndexOutOfBoundsException(); + } + if (fromIndex >= toIndex) { + for (int index = 0; index < count; index++) { + UpdateGlobalRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + *ArrayAddressOfElementAt(array, fromIndex + index)); } - // TODO: refcounting! - memmove(ArrayAddressOfElementAt(destinationArray, toIndex), - ArrayAddressOfElementAt(array, fromIndex), count * sizeof(KRef)); + } else { + for (int index = count - 1; index >= 0; index--) { + UpdateGlobalRef(ArrayAddressOfElementAt(destinationArray, toIndex + index), + *ArrayAddressOfElementAt(array, fromIndex + index)); + } + } } // Arrays.kt diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index c25cbe44220..2a64f890bc4 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -1,23 +1,82 @@ -#include #include +#include +#include #include // for offsetof +#include // only for memory tracing. +#include #include "Assert.h" #include "Exceptions.h" #include "Memory.h" #include "Natives.h" -void FreeObject(ContainerHeader* header) { +// Define to 1 to see all memory operations. +#define TRACE_MEMORY 0 +// Define to 1 to use in multithreaded environment. +#define CONCURRENT 0 + +namespace { + +// Current number of allocated containers. +int allocCount = 0; +#if TRACE_MEMORY +// List of all global objects addresses. +std::vector* globalObjects = nullptr; +// Set of all containers. +std::set* containers = nullptr; +#endif + +} // namespace + +ContainerHeader* AllocContainer(size_t size) { + ContainerHeader* result = reinterpret_cast(calloc(1, size)); +#if TRACE_MEMORY + printf(">>> alloc %d -> %p\n", (int)size, result); + containers->insert(result); +#endif + // TODO: atomic increment in concurrent case. + allocCount++; + return result; +} + +void FreeContainer(ContainerHeader* header) { +#if TRACE_MEMORY + printf("<<< free %p\n", header); + containers->erase(header); +#endif header->ref_count_ = CONTAINER_TAG_INVALID; + + // Now let's clean all object's fields in this container. + // TODO: this is gross hack, relying on the fact that we now only alloc + // ArenaContainer and ObjectContainer, which both have single element. + ObjHeader* obj = reinterpret_cast(header + 1); + const TypeInfo* typeInfo = obj->type_info(); + + for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { + ObjHeader** location = reinterpret_cast( + reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); + UpdateGlobalRef(location, nullptr); + } + // Object arrays are *special*. + if (typeInfo == theArrayTypeInfo) { + ArrayHeader* array = obj->array(); + for (int index = 0; index < array->count_; index++) { + UpdateGlobalRef(ArrayAddressOfElementAt(array, index), nullptr); + } + } + + // And release underlying memory. + // TODO: atomic decrement in concurrent case. + allocCount--; free(header); } ArenaContainer::ArenaContainer(uint32_t size) { - ArenaContainerHeader* header = reinterpret_cast( - calloc(size + sizeof(ArenaContainerHeader), 1)); + ArenaContainerHeader* header = + static_cast(AllocContainer(size + sizeof(ArenaContainerHeader))); header_ = header; - header->ref_count_ = CONTAINER_TAG_INCREMENT; + // header->ref_count_ is zero initialized by AllocContainer(). header->current_ = reinterpret_cast(header_) + sizeof(ArenaContainerHeader); header->end_ = header->current_ + size; @@ -25,12 +84,15 @@ ArenaContainer::ArenaContainer(uint32_t size) { void ObjectContainer::Init(const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object"); - uint32_t alloc_size = + uint32_t alloc_size = sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_; - header_ = reinterpret_cast(calloc(alloc_size, 1)); + header_ = AllocContainer(alloc_size); if (header_) { - header_->ref_count_ = CONTAINER_TAG_INCREMENT; + // header->ref_count_ is zero initialized by AllocContainer(). SetMeta(GetPlace(), type_info); +#if TRACE_MEMORY + printf("object at %p\n", GetPlace()); +#endif } } @@ -39,12 +101,15 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) { uint32_t alloc_size = sizeof(ContainerHeader) + sizeof(ArrayHeader) - type_info->instanceSize_ * elements; - header_ = reinterpret_cast(calloc(alloc_size, 1)); + header_ = AllocContainer(alloc_size); RuntimeAssert(header_ != nullptr, "Cannot alloc memory"); if (header_) { - header_->ref_count_ = CONTAINER_TAG_INCREMENT; + // header->ref_count_ is zero initialized by AllocContainer(). GetPlace()->count_ = elements; SetMeta(GetPlace()->obj(), type_info); +#if TRACE_MEMORY + printf("array at %p\n", GetPlace()); +#endif } } @@ -71,6 +136,20 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, int count) { return result; } +inline void AddRef(const ObjHeader* object) { +#if TRACE_MEMORY + printf("AddRef on %p in %p\n", object, object->container()); +#endif + AddRef(object->container()); +} + +inline void ReleaseRef(const ObjHeader* object) { +#if TRACE_MEMORY + printf("ReleaseRef on %p in %p\n", object, object->container()); +#endif + Release(object->container()); +} + #ifdef __cplusplus extern "C" { #endif @@ -85,6 +164,35 @@ void InitMemory() { offsetof(ObjHeader , container_offset_negative_), "Layout mismatch"); // TODO: initialize heap here. + allocCount = 0; +#if TRACE_MEMORY + globalObjects = new std::vector(); + containers = new std::set(); +#endif +} + +void DeinitMemory() { +#if TRACE_MEMORY + // Free all global objects, to ensure no memory leaks happens. + for (auto location: *globalObjects) { + printf("Release global in *%p: %p\n", location, *location); + UpdateGlobalRef(location, nullptr); + } + delete globalObjects; + globalObjects = nullptr; +#endif + + if (allocCount > 0) { +#if TRACE_MEMORY + // TODO: move out of TRACE_MEMORY, once exceptions free memory. + printf("*** Memory leaks, leaked %d containers ***\n", allocCount); + for (auto container: *containers) { + printf("Unfreed container %p, count = %d\n", container, container->ref_count_); + } + delete containers; + containers = nullptr; +#endif + } } // Now we ignore all placement hints and always allocate heap space for new object. @@ -115,6 +223,7 @@ OBJ_GETTER(InitInstance, ObjHeader* sentinel = reinterpret_cast(1); ObjHeader* value; // Wait until other initializers. + // TODO: check CONCURRENT! while ((value = __sync_val_compare_and_swap( location, nullptr, sentinel)) == sentinel) { // TODO: consider yielding. @@ -126,28 +235,88 @@ OBJ_GETTER(InitInstance, } AllocInstance(type_info, hint, OBJ_RESULT); + ObjHeader* object = *OBJ_RESULT; try { - ctor(*OBJ_RESULT); - bool ok = __sync_bool_compare_and_swap(location, sentinel, *OBJ_RESULT); - RuntimeAssert(ok, "CAS must succeed"); + ctor(object); + UpdateGlobalRef(location, object); +#if CONCURRENT + // TODO: locking or smth lock-free in MT case? +#endif +#if TRACE_MEMORY + globalObjects->push_back(location); +#endif RETURN_OBJ_RESULT(); } catch (...) { - __sync_val_compare_and_swap(location, sentinel, nullptr); + UpdateLocalRef(OBJ_RESULT, nullptr); + UpdateGlobalRef(location, nullptr); RETURN_OBJ(nullptr); } } -// Just stubs. void SetLocalRef(ObjHeader** location, const ObjHeader* object) { +#if TRACE_MEMORY + printf("SetLocalRef *%p: %p\n", location, object); +#endif *const_cast(location) = object; + if (object != nullptr) { + AddRef(object); + } +} + +void SetGlobalRef(ObjHeader** location, const ObjHeader* object) { +#if TRACE_MEMORY + printf("SetGlobalRef *%p: %p\n", location, object); +#endif + *const_cast(location) = object; + if (object != nullptr) { + AddRef(object); + } +#if CONCURRENT + // TODO: memory fence here. +#endif } void UpdateLocalRef(ObjHeader** location, const ObjHeader* object) { - *const_cast(location) = object; + ObjHeader* old = *location; +#if TRACE_MEMORY + printf("UpdateLocalRef *%p: %p -> %p\n", location, old, object); +#endif + if (old != object) { + *const_cast(location) = object; + if (old > reinterpret_cast(1)) { + ReleaseRef(old); + } + if (object != nullptr) { + AddRef(object); + } + } } void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) { - *const_cast(location) = object; +#if CONCURRENT + ObjHeader* old = *location; +#if TRACE_MEMORY + printf("UpdateGlobalRef *%p: %p -> %p\n", location, old, object); +#endif + if (old != object) { + if (object != nullptr) { + AddRef(object); + } + bool written = __sync_bool_compare_and_swap( + location, old, const_cast(object)); + if (written) { + if (old > reinterpret_cast(1)) { + ReleaseRef(old); + } + } else { + if (object != nullptr) { + ReleaseRef(object); + } + } + } +#else + UpdateLocalRef(location, object); +#endif } #ifdef __cplusplus diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 7fdad06ff5a..7927862edfe 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -10,7 +10,9 @@ typedef enum { // Allocation is generic global allocation. SCOPE_GLOBAL = 1, // Allocation shall take place in current arena. - SCOPE_ARENA = 2 + SCOPE_ARENA = 2, + // Allocation is permanent. + SCOPE_PERMANENT = 3 } PlacementHint; // Must fit in two bits. @@ -116,6 +118,8 @@ inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { return -obj->type_info()->instanceSize_ * obj->count_; } +void FreeContainer(ContainerHeader* header); + // Those two operations are implemented by translator when storing references // to objects. inline void AddRef(ContainerHeader* header) { @@ -136,13 +140,11 @@ inline void AddRef(ContainerHeader* header) { } } -void FreeObject(ContainerHeader* header); - inline void Release(ContainerHeader* header) { switch (header->ref_count_ & CONTAINER_TAG_MASK) { case CONTAINER_TAG_NORMAL: if ((header->ref_count_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) { - FreeObject(header); + FreeContainer(header); } break; case CONTAINER_TAG_NOCOUNT: @@ -163,7 +165,7 @@ inline void Release(ContainerHeader* header) { case CONTAINER_TAG_SHARED: if (__sync_sub_and_fetch( &header->ref_count_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) { - FreeObject(header); + FreeContainer(header); } break; case CONTAINER_TAG_INVALID: @@ -182,6 +184,7 @@ class Container { obj->container_offset_negative_ = reinterpret_cast(obj) - reinterpret_cast(header_); obj->set_type_info(type_info); + RuntimeAssert(obj->container() == header_, "Placement must match"); } public: @@ -207,7 +210,8 @@ class ObjectContainer : public Container { Init(type_info); } - // Object container shalln't have any dtor, as it's being freed by ::Release(). + // Object container shalln't have any dtor, as it's being freed by + // ::Release(). ObjHeader* GetPlace() const { return reinterpret_cast( reinterpret_cast(header_) + sizeof(ContainerHeader)); @@ -272,7 +276,7 @@ class ArenaContainer : public Container { // Dispose whole container ignoring non-zero refcount. Use with care. void Dispose() { if (header_) { - FreeObject(header_); + FreeContainer(header_); header_ = nullptr; } } @@ -285,12 +289,14 @@ extern "C" { #define OBJ_RESULT __result__ #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT) #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT) -#define RETURN_OBJ(value) UpdateLocalRef(OBJ_RESULT, value); return value; +#define RETURN_OBJ(value) { ObjHeader* obj = value; UpdateLocalRef(OBJ_RESULT, obj); return obj; } #define RETURN_OBJ_RESULT() return *OBJ_RESULT; #define RETURN_RESULT_OF0(name) name(OBJ_RESULT); return *OBJ_RESULT; #define RETURN_RESULT_OF(name, ...) name(__VA_ARGS__, OBJ_RESULT); return *OBJ_RESULT; void InitMemory(); +void DeinitMemory(); + OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint); OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, PlacementHint hint, uint32_t elements); @@ -309,6 +315,37 @@ void UpdateLocalRef(ObjHeader** location, const ObjHeader* object); // Update potentially globally visible location. void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object); +// +// Object reference management. +// +// Reference management scheme we use assumes significant degree of flexibility, so that +// one could implement either pure reference counting scheme, or tracing collector without +// much ado. +// Most important primitive is UpdateRef() API, which modifies location to use new +// object reference. In pure reference counted scheme it will check old value, +// decrement reference, increment counter on the new value, and store it into the field. +// In tracing collector-like scheme, only field updates counts, and all other operations are +// essentially no-ops. +// +// On codegeneration phase we adopt following approaches: +// - every stack frame has several slots, holding object references (allRefs) +// - those are known by compiler (and shall be grouped together) +// - it keeps all locally allocated objects in such slot +// - all local variables keeping an object also allocate a slot +// - most manipulations on objects happens in SSA variables and do no affect slots +// - exception handlers knowns slot locations for every function, and can update references +// in intermediate frames when throwing +// + +// Sets locally visible location. +void SetLocalRef(ObjHeader** location, const ObjHeader* object); +// Sets potentially globally visible location. +void SetGlobalRef(ObjHeader** location, const ObjHeader* object); +// Update locally visible location. +void UpdateLocalRef(ObjHeader** location, const ObjHeader* object); +// Update potentially globally visible location. +void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object); + #ifdef __cplusplus } #endif diff --git a/runtime/src/main/cpp/Types.cpp b/runtime/src/main/cpp/Types.cpp index 3724e9dd8a9..f5c91305151 100644 --- a/runtime/src/main/cpp/Types.cpp +++ b/runtime/src/main/cpp/Types.cpp @@ -24,6 +24,11 @@ KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info) { return obj_type_info != nullptr; } +KBoolean IsArray(KConstRef obj) { + RuntimeAssert(obj != nullptr, "Object must not be null"); + return obj->type_info()->instanceSize_ < 0; +} + void CheckInstance(const ObjHeader* obj, const TypeInfo* type_info) { if (IsInstance(obj, type_info)) { return; diff --git a/runtime/src/main/cpp/Types.h b/runtime/src/main/cpp/Types.h index 8f1d23b5ed2..4ee1a02aa1f 100644 --- a/runtime/src/main/cpp/Types.h +++ b/runtime/src/main/cpp/Types.h @@ -38,6 +38,7 @@ extern const TypeInfo* theThrowableTypeInfo; KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info); void CheckCast(const ObjHeader* obj, const TypeInfo* type_info); +KBoolean IsArray(KConstRef obj); typedef void (*Initializer)(); struct InitNode {