diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index 60587c9cd9a..37cc9dabd8f 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -109,7 +109,9 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, get() = (functionDescriptor as? ClassConstructorDescriptor)?.constructedClass private var returnSlot: LLVMValueRef? = null private var slotsPhi: LLVMValueRef? = null - private var slotCount = 1 + private val frameOverlaySlotCount = + (LLVMStoreSizeOfType(llvmTargetData, runtime.frameOverlayType) / pointerSize).toInt() + private var slotCount = frameOverlaySlotCount private var localAllocs = 0 private var arenaSlot: LLVMValueRef? = null private val slotToVariableLocation = mutableMapOf() @@ -227,10 +229,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, call(context.llvm.updateReturnRefFunction, listOf(address, value)) } - // Only use ignoreOld, when sure that memory is freshly inited and have no value. - private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) { - call(if (ignoreOld) context.llvm.setRefFunction else context.llvm.updateRefFunction, - listOf(address, value)) + private fun updateRef(value: LLVMValueRef, address: LLVMValueRef) { + call(context.llvm.updateRefFunction, listOf(address, value)) } //-------------------------------------------------------------------------// @@ -502,6 +502,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, listOf(slotsMem, Int8(0).llvm, Int32(slotCount * pointerSize).llvm, Int32(alignment).llvm, Int1(0).llvm)) + call(context.llvm.enterFrameFunction, listOf(slots, Int32(slotCount).llvm)) } addPhiIncoming(slotsPhi!!, prologueBb to slots) val slotOffset = pointerSize * slotCount @@ -657,7 +658,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, private val needSlots: Boolean get() { - return slotCount > 1 || localAllocs > 0 || + return slotCount > frameOverlaySlotCount || localAllocs > 0 || // Prevent empty cleanup on mingw to workaround LLVM bug: context.config.targetManager.target == KonanTarget.MINGW } diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index d3708ed430c..9252103c16c 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -315,8 +315,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { val allocArrayFunction = importRtFunction("AllocArrayInstance") val initInstanceFunction = importRtFunction("InitInstance") val updateReturnRefFunction = importRtFunction("UpdateReturnRef") - val setRefFunction = importRtFunction("SetRef") val updateRefFunction = importRtFunction("UpdateRef") + val enterFrameFunction = importRtFunction("EnterFrame") val leaveFrameFunction = importRtFunction("LeaveFrame") val getReturnSlotIfArenaFunction = importRtFunction("GetReturnSlotIfArena") val getParamSlotIfArenaFunction = importRtFunction("GetParamSlotIfArena") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt index d1a76271728..63b8389aa61 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt @@ -40,6 +40,8 @@ class Runtime(bitcodeFile: String) { val objHeaderPtrType = pointerType(objHeaderType) val arrayHeaderType = getStructType("ArrayHeader") + val frameOverlayType = getStructType("FrameOverlay") + val target = LLVMGetTarget(llvmModule)!!.toKString() // TODO: deduce TLS model from explicit config parameter. diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 5e85a576370..995f9406304 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -46,8 +46,6 @@ constexpr container_size_t kContainerAlignment = 1024; // Single object alignment. constexpr container_size_t kObjectAlignment = 8; -} // namespace - #if USE_GC // Collection threshold default (collect after having so many elements in the // release candidates set). Better be a prime number. @@ -56,12 +54,18 @@ constexpr size_t kGcThreshold = 9341; typedef KStdDeque ContainerHeaderDeque; #endif +} // namespace + #if TRACE_MEMORY || USE_GC typedef KStdUnorderedSet ContainerHeaderSet; typedef KStdVector ContainerHeaderList; typedef KStdVector KRefPtrList; #endif +struct FrameOverlay { + ArenaContainer* arena; +}; + struct MemoryState { // Current number of allocated containers. int allocCount = 0; @@ -99,6 +103,8 @@ namespace { // TODO: can we pass this variable as an explicit argument? THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr; +constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**); + inline bool isFreeable(const ContainerHeader* header) { return (header->refCount_ & CONTAINER_TAG_MASK) < CONTAINER_TAG_PERMANENT; } @@ -131,6 +137,15 @@ inline ObjHeader** asArenaSlot(ObjHeader** slot) { reinterpret_cast(slot) & ~ARENA_BIT); } +inline FrameOverlay* asFrameOverlay(ObjHeader** slot) { + return reinterpret_cast(slot); +} + +inline bool isRefCounted(KConstRef object) { + return (object->container()->refCount_ & CONTAINER_TAG_MASK) == + CONTAINER_TAG_NORMAL; +} + #if USE_GC inline void processFinalizerQueue(MemoryState* state) { @@ -777,27 +792,12 @@ ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) { void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { if (isArenaSlot(returnSlot)) { - if (object == nullptr - || (object->container()->refCount_ & CONTAINER_TAG_MASK) > CONTAINER_TAG_NORMAL) { - // Not a subject of reference counting. - return; - } + // Not a subject of reference counting. + if (object == nullptr || !isRefCounted(object)) return; auto arena = initedArena(asArenaSlot(returnSlot)); returnSlot = arena->getSlot(); } - ObjHeader* old = *returnSlot; -#if TRACE_MEMORY - fprintf(stderr, "UpdateReturnRef *%p: %p -> %p\n", returnSlot, old, object); -#endif - if (old != object) { - if (object != nullptr) { - AddRef(object); - } - *const_cast(returnSlot) = object; - if (old > reinterpret_cast(1)) { - ReleaseRef(old); - } - } + UpdateRef(returnSlot, object); } void UpdateRef(ObjHeader** location, const ObjHeader* object) { @@ -817,11 +817,17 @@ void UpdateRef(ObjHeader** location, const ObjHeader* object) { } } +void EnterFrame(ObjHeader** start, int count) { +#if TRACE_MEMORY + fprintf(stderr, "EnterFrame %p .. %p\n", start, start + count); +#endif +} + void LeaveFrame(ObjHeader** start, int count) { #if TRACE_MEMORY - fprintf(stderr, "LeaveFrame %p .. %p\n", start, start + count); + fprintf(stderr, "LeaveFrame %p .. %p\n", start, start + count); #endif - ReleaseRefs(start + 1, count - 1); + ReleaseRefs(start + kFrameOverlaySlots, count - kFrameOverlaySlots); if (*start != nullptr) { auto arena = initedArena(start); #if TRACE_MEMORY @@ -854,10 +860,11 @@ void GarbageCollect() { RuntimeAssert(state->toFree != nullptr, "GC must not be stopped"); RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed"); + state->gcInProgress = true; + // Flush cache. flushFreeableCache(state); - state->gcInProgress = true; // Traverse inner pointers in the closure of release candidates, and // temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing. #if TRACE_GC_PHASES diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index bf588c81a70..704f2682e70 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -397,6 +397,8 @@ void UpdateRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW; void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW; // Optimization: release all references in range. void ReleaseRefs(ObjHeader** start, int count) RUNTIME_NOTHROW; +// Called on frame enter, if it has object slots. +void EnterFrame(ObjHeader** start, int count) RUNTIME_NOTHROW; // Called on frame leave, if it has object slots. void LeaveFrame(ObjHeader** start, int count) RUNTIME_NOTHROW; // Tries to use returnSlot's arena for allocation. diff --git a/runtime/src/main/cpp/Natives.h b/runtime/src/main/cpp/Natives.h index 523b34e776e..1a85e9af8ad 100644 --- a/runtime/src/main/cpp/Natives.h +++ b/runtime/src/main/cpp/Natives.h @@ -134,6 +134,9 @@ OBJ_GETTER0(Kotlin_getCurrentStackTrace); OBJ_GETTER0(Kotlin_konan_internal_undefined); +void Kotlin_konan_internal_GC_suspend(KRef); +void Kotlin_konan_internal_GC_resume(KRef); + #ifdef __cplusplus } #endif diff --git a/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt b/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt index de87f832398..a4f64fd6bd3 100644 --- a/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt +++ b/runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt @@ -17,6 +17,7 @@ package konan.internal import kotlin.internal.getProgressionLastElement +import kotlin.text.toUtf8Array @ExportForCppRuntime fun ThrowNullPointerException(): Nothing { @@ -101,3 +102,21 @@ fun getProgressionLast(start: Char, end: Char, step: Int): Char = fun getProgressionLast(start: Int, end: Int, step: Int): Int = getProgressionLastElement(start, end, step) fun getProgressionLast(start: Long, end: Long, step: Long): Long = getProgressionLastElement(start, end, step) + +// Called by the debugger. +@ExportForCppRuntime +fun KonanObjectToUtf8Array(value: Any?): ByteArray { + val string = when (value) { + is Array<*> -> value.contentToString() + is CharArray -> value.contentToString() + is BooleanArray -> value.contentToString() + is ByteArray -> value.contentToString() + is ShortArray -> value.contentToString() + is IntArray -> value.contentToString() + is LongArray -> value.contentToString() + is FloatArray -> value.contentToString() + is DoubleArray -> value.contentToString() + else -> value.toString() + } + return toUtf8Array(string, 0, string.length) +} diff --git a/runtime/src/main/kotlin/konan/internal/Util.kt b/runtime/src/main/kotlin/konan/internal/Util.kt deleted file mode 100644 index 364391e2e29..00000000000 --- a/runtime/src/main/kotlin/konan/internal/Util.kt +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2010-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package konan.internal - -import kotlin.text.toUtf8Array - -// Called by debugger. -@ExportForCppRuntime -fun KonanObjectToUtf8Array(value: Any?): ByteArray { - val string = when (value) { - is Array<*> -> value.contentToString() - is CharArray -> value.contentToString() - is BooleanArray -> value.contentToString() - is ByteArray -> value.contentToString() - is ShortArray -> value.contentToString() - is IntArray -> value.contentToString() - is LongArray -> value.contentToString() - is FloatArray -> value.contentToString() - is DoubleArray -> value.contentToString() - else -> value.toString() - } - return toUtf8Array(string, 0, string.length) -} \ No newline at end of file