diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index caf8fbc4940..4de0469baec 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -18,6 +18,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { var constructedClass: ClassDescriptor? = null val vars = VariableManager(this) var returnSlot: LLVMValueRef? = null + var slotsPhi: LLVMValueRef? = null + var slotCount = 0 fun prologue(descriptor: FunctionDescriptor) { prologue(llvmFunction(descriptor), @@ -41,17 +43,34 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { entryBb = LLVMAppendBasicBlock(function, "entry") epilogueBb = LLVMAppendBasicBlock(function, "epilogue") positionAtEnd(entryBb!!) + slotsPhi = phi(kObjHeaderPtrPtr) + slotCount = 0 } fun epilogue() { appendingTo(prologueBb!!) { + val slots = if (slotCount > 0) + LLVMBuildArrayAlloca(builder, kObjHeaderPtr, Int32(slotCount).llvm, "")!! + else + kNullObjHeaderPtrPtr + if (slotCount > 0) { + // Zero-init slots. + val slotsMem = bitcast(kInt8Ptr, slots) + val pointerSize = LLVMABISizeOfType(llvmTargetData, kObjHeaderPtr).toInt() + val alignment = LLVMABIAlignmentOfType(llvmTargetData, kObjHeaderPtr) + call(context.llvm.memsetFunction, + listOf(slotsMem, Int8(0).llvm, + Int32(slotCount * pointerSize).llvm, Int32(alignment).llvm, + Int1(0).llvm)) + } + addPhiIncoming(slotsPhi!!, prologueBb!! to slots) br(entryBb!!) } appendingTo(epilogueBb!!) { when { returnType == voidType -> { - vars.releaseVars() + releaseVars() assert(returnSlot == null) LLVMBuildRetVoid(builder) } @@ -61,7 +80,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { if (returnSlot != null) { updateLocalRef(returnPhi, returnSlot!!) } - vars.releaseVars() + releaseVars() LLVMBuildRet(builder, returnPhi) } // Do nothing, all paths throw. @@ -72,6 +91,14 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { returns.clear() vars.clear() returnSlot = null + slotsPhi = null + } + + fun releaseVars() { + if (slotCount > 0) { + call(context.llvm.releaseLocalRefsFunction, + listOf(slotsPhi, Int32(slotCount).llvm)) + } } private var prologueBb: LLVMBasicBlockRef? = null @@ -104,11 +131,11 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun intToPtr(imm: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, imm, DestTy, Name)!! fun alloca(type: LLVMTypeRef?, name: String = ""): LLVMValueRef { + if (isObjectType(type!!)) { + return gep(slotsPhi!!, Int32(slotCount++).llvm) + } appendingTo(prologueBb!!) { - val result = LLVMBuildAlloca(builder, type, name)!! - if (isObjectType(type!!)) - LLVMBuildStore(builder, kNullObjHeaderPtr, result) - return result + return LLVMBuildAlloca(builder, type, name)!! } } fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildLoad(builder, value, name)!! @@ -132,6 +159,12 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { } } + fun gep(base: LLVMValueRef, index: LLVMValueRef): LLVMValueRef { + memScoped { + val args = allocArrayOf(index) + return LLVMBuildGEP(builder, base, args[0].ptr, 1, "")!! + } + } // Only use ignoreOld, when sure that memory is freshly inited and have no value. fun updateLocalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) { diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 423ead8ec1d..e0442f29b06 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -205,6 +205,14 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { return LLVMAddFunction(llvmModule, name, functionType)!! } + private fun importMemset() : LLVMValueRef { + memScoped { + val parameterTypes = allocArrayOf(int8TypePtr, int8Type, int32Type, int32Type, int1Type) + val functionType = LLVMFunctionType(LLVMVoidType(), parameterTypes[0].ptr, 5, 0) + return LLVMAddFunction(llvmModule, "llvm.memset.p0i8.i32", functionType)!! + } + } + val staticData = StaticData(context) val runtimeFile = context.config.configuration.get(KonanConfigKeys.RUNTIME_FILE)!! @@ -226,6 +234,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val setGlobalRefFunction = importRtFunction("SetGlobalRef") val updateLocalRefFunction = importRtFunction("UpdateLocalRef") val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef") + val releaseLocalRefsFunction = importRtFunction("ReleaseLocalRefs") val setArrayFunction = importRtFunction("Kotlin_Array_set") val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl") val lookupFieldOffset = importRtFunction("LookupFieldOffset") @@ -234,6 +243,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val checkInstanceFunction = importRtFunction("CheckInstance") val throwExceptionFunction = importRtFunction("ThrowException") val appendToInitalizersTail = importRtFunction("AppendToInitializersTail") + val memsetFunction = importMemset() val usedFunctions = mutableListOf() val staticInitializers = mutableListOf() val fileInitializers = mutableListOf() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt index d7989ce4ca5..a0b772b0fc1 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt @@ -68,6 +68,10 @@ internal open class Struct(val type: LLVMTypeRef?, val elements: List, value: LLVMValueRef? = null) : Int { // Note that we always create slot for object references for memory management. val descriptor = scoped.first diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 289ede0de8d..6ee6fd74ee8 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -55,17 +55,16 @@ void FreeContainer(ContainerHeader* header) { ObjHeader* obj = reinterpret_cast(header + 1); const TypeInfo* typeInfo = obj->type_info(); + // We use *local* versions as no other threads could see dead objects. for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { ObjHeader** location = reinterpret_cast( reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); - UpdateGlobalRef(location, nullptr); + UpdateLocalRef(location, nullptr); } // Object arrays are *special*. if (typeInfo == theArrayTypeInfo) { ArrayHeader* array = obj->array(); - for (int index = 0; index < array->count_; index++) { - UpdateGlobalRef(ArrayAddressOfElementAt(array, index), nullptr); - } + ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_); } // And release underlying memory. @@ -152,9 +151,7 @@ inline void ReleaseRef(const ObjHeader* object) { Release(object->container()); } -#ifdef __cplusplus extern "C" { -#endif void InitMemory() { RuntimeAssert(offsetof(ArrayHeader, type_info_) @@ -321,6 +318,50 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) { #endif } -#ifdef __cplusplus -} +void ReleaseLocalRefs(ObjHeader** start, int count) { +#if TRACE_MEMORY + printf("ReleaseLocalRefs %p .. %p\n", start, start + count); #endif + ObjHeader** current = start; + while (count-- > 0) { + ObjHeader* object = *current; + if (object != nullptr) { + ReleaseRef(object); + // Just for sanity, optional. + *current = nullptr; + } + current++; + } +} + +void ReleaseGlobalRefs(ObjHeader** start, int count) { +#if TRACE_MEMORY + printf("ReleaseGlobalRefs %p .. %p\n", start, start + count); +#endif +#if CONCURRENT + ObjHeader** current = start; + while (count-- > 0) { + ObjHeader* object = *current; + if (object != nullptr) { + bool written = __sync_bool_compare_and_swap( + current, object, nullptr); + if (written) + ReleaseRef(object); + } + current++; + } +#else + ObjHeader** current = start; + while (count-- > 0) { + ObjHeader* object = *current; + if (object != nullptr) { + ReleaseRef(object); + // Usually required. + *current = nullptr; + } + current++; + } +#endif +} + +} // extern "C" diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index e888f2ed2b9..9cb5a8f56d4 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -351,6 +351,9 @@ void SetGlobalRef(ObjHeader** location, const ObjHeader* object); void UpdateLocalRef(ObjHeader** location, const ObjHeader* object); // Update potentially globally visible location. void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object); +// Optimization: release all references in range. +void ReleaseLocalRefs(ObjHeader** start, int count); +void ReleaseGlobalRefs(ObjHeader** start, int count); #ifdef __cplusplus }