Added FrameOverlay struct

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -109,7 +109,9 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         get() = (functionDescriptor as? ClassConstructorDescriptor)?.constructedClass
     private var returnSlot: LLVMValueRef? = null
     private var slotsPhi: LLVMValueRef? = null
-    private var slotCount = 1
+    private val frameOverlaySlotCount =
+            (LLVMStoreSizeOfType(llvmTargetData, runtime.frameOverlayType) / pointerSize).toInt()
+    private var slotCount = frameOverlaySlotCount
     private var localAllocs = 0
     private var arenaSlot: LLVMValueRef? = null
     private val slotToVariableLocation = mutableMapOf<Int,VariableDebugLocation>()
@@ -227,10 +229,8 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         call(context.llvm.updateReturnRefFunction, listOf(address, value))
     }
 
-    // Only use ignoreOld, when sure that memory is freshly inited and have no value.
-    private fun updateRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) {
-        call(if (ignoreOld) context.llvm.setRefFunction else context.llvm.updateRefFunction,
-                listOf(address, value))
+    private fun updateRef(value: LLVMValueRef, address: LLVMValueRef) {
+        call(context.llvm.updateRefFunction, listOf(address, value))
     }
 
     //-------------------------------------------------------------------------//
@@ -502,6 +502,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
                         listOf(slotsMem, Int8(0).llvm,
                                 Int32(slotCount * pointerSize).llvm, Int32(alignment).llvm,
                                 Int1(0).llvm))
+                call(context.llvm.enterFrameFunction, listOf(slots, Int32(slotCount).llvm))
             }
             addPhiIncoming(slotsPhi!!, prologueBb to slots)
             val slotOffset = pointerSize * slotCount
@@ -657,7 +658,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
 
     private val needSlots: Boolean
         get() {
-            return slotCount > 1 || localAllocs > 0 ||
+            return slotCount > frameOverlaySlotCount || localAllocs > 0 ||
                     // Prevent empty cleanup on mingw to workaround LLVM bug:
                     context.config.targetManager.target == KonanTarget.MINGW
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -315,8 +315,8 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val allocArrayFunction = importRtFunction("AllocArrayInstance")
     val initInstanceFunction = importRtFunction("InitInstance")
     val updateReturnRefFunction = importRtFunction("UpdateReturnRef")
-    val setRefFunction = importRtFunction("SetRef")
     val updateRefFunction = importRtFunction("UpdateRef")
+    val enterFrameFunction = importRtFunction("EnterFrame")
     val leaveFrameFunction = importRtFunction("LeaveFrame")
     val getReturnSlotIfArenaFunction = importRtFunction("GetReturnSlotIfArena")
     val getParamSlotIfArenaFunction = importRtFunction("GetParamSlotIfArena")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt

@@ -40,6 +40,8 @@ class Runtime(bitcodeFile: String) {
     val objHeaderPtrType = pointerType(objHeaderType)
     val arrayHeaderType = getStructType("ArrayHeader")
 
+    val frameOverlayType = getStructType("FrameOverlay")
+
     val target = LLVMGetTarget(llvmModule)!!.toKString()
 
     // TODO: deduce TLS model from explicit config parameter.

runtime/src/main/cpp/Memory.cpp

@@ -46,8 +46,6 @@ constexpr container_size_t kContainerAlignment = 1024;
 // Single object alignment.
 constexpr container_size_t kObjectAlignment = 8;
 
-}  // namespace
-
 #if USE_GC
 // Collection threshold default (collect after having so many elements in the
 // release candidates set). Better be a prime number.
@@ -56,12 +54,18 @@ constexpr size_t kGcThreshold = 9341;
 typedef KStdDeque<ContainerHeader*> ContainerHeaderDeque;
 #endif
 
+}  // namespace
+
 #if TRACE_MEMORY || USE_GC
 typedef KStdUnorderedSet<ContainerHeader*> ContainerHeaderSet;
 typedef KStdVector<ContainerHeader*> ContainerHeaderList;
 typedef KStdVector<KRef*> KRefPtrList;
 #endif
 
+struct FrameOverlay {
+  ArenaContainer* arena;
+};
+
 struct MemoryState {
   // Current number of allocated containers.
   int allocCount = 0;
@@ -99,6 +103,8 @@ namespace {
 // TODO: can we pass this variable as an explicit argument?
 THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr;
 
+constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**);
+
 inline bool isFreeable(const ContainerHeader* header) {
   return (header->refCount_ & CONTAINER_TAG_MASK) < CONTAINER_TAG_PERMANENT;
 }
@@ -131,6 +137,15 @@ inline ObjHeader** asArenaSlot(ObjHeader** slot) {
       reinterpret_cast<uintptr_t>(slot) & ~ARENA_BIT);
 }
 
+inline FrameOverlay* asFrameOverlay(ObjHeader** slot) {
+  return reinterpret_cast<FrameOverlay*>(slot);
+}
+
+inline bool isRefCounted(KConstRef object) {
+  return (object->container()->refCount_ & CONTAINER_TAG_MASK) ==
+      CONTAINER_TAG_NORMAL;
+}
+
 #if USE_GC
 
 inline void processFinalizerQueue(MemoryState* state) {
@@ -777,27 +792,12 @@ ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) {
 
 void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
   if (isArenaSlot(returnSlot)) {
-    if (object == nullptr
-        || (object->container()->refCount_ & CONTAINER_TAG_MASK) > CONTAINER_TAG_NORMAL) {
-        // Not a subject of reference counting.
-        return;
-    }
+    // Not a subject of reference counting.
+    if (object == nullptr || !isRefCounted(object)) return;
     auto arena = initedArena(asArenaSlot(returnSlot));
     returnSlot = arena->getSlot();
   }
-  ObjHeader* old = *returnSlot;
-#if TRACE_MEMORY
-  fprintf(stderr, "UpdateReturnRef *%p: %p -> %p\n", returnSlot, old, object);
-#endif
-  if (old != object) {
-    if (object != nullptr) {
-      AddRef(object);
-    }
-    *const_cast<const ObjHeader**>(returnSlot) = object;
-    if (old > reinterpret_cast<ObjHeader*>(1)) {
-      ReleaseRef(old);
-    }
-  }
+  UpdateRef(returnSlot, object);
 }
 
 void UpdateRef(ObjHeader** location, const ObjHeader* object) {
@@ -817,11 +817,17 @@ void UpdateRef(ObjHeader** location, const ObjHeader* object) {
   }
 }
 
+void EnterFrame(ObjHeader** start, int count) {
+#if TRACE_MEMORY
+  fprintf(stderr, "EnterFrame %p .. %p\n", start, start + count);
+#endif
+}
+
 void LeaveFrame(ObjHeader** start, int count) {
 #if TRACE_MEMORY
-    fprintf(stderr, "LeaveFrame %p .. %p\n", start, start + count);
+  fprintf(stderr, "LeaveFrame %p .. %p\n", start, start + count);
 #endif
-  ReleaseRefs(start + 1, count - 1);
+  ReleaseRefs(start + kFrameOverlaySlots, count - kFrameOverlaySlots);
   if (*start != nullptr) {
     auto arena = initedArena(start);
 #if TRACE_MEMORY
@@ -854,10 +860,11 @@ void GarbageCollect() {
   RuntimeAssert(state->toFree != nullptr, "GC must not be stopped");
   RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed");
 
+  state->gcInProgress = true;
+
   // Flush cache.
   flushFreeableCache(state);
 
-  state->gcInProgress = true;
   // Traverse inner pointers in the closure of release candidates, and
   // temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing.
 #if TRACE_GC_PHASES

runtime/src/main/cpp/Memory.h

@@ -397,6 +397,8 @@ void UpdateRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW;
 // Optimization: release all references in range.
 void ReleaseRefs(ObjHeader** start, int count) RUNTIME_NOTHROW;
+// Called on frame enter, if it has object slots.
+void EnterFrame(ObjHeader** start, int count) RUNTIME_NOTHROW;
 // Called on frame leave, if it has object slots.
 void LeaveFrame(ObjHeader** start, int count) RUNTIME_NOTHROW;
 // Tries to use returnSlot's arena for allocation.

runtime/src/main/cpp/Natives.h

@@ -134,6 +134,9 @@ OBJ_GETTER0(Kotlin_getCurrentStackTrace);
 
 OBJ_GETTER0(Kotlin_konan_internal_undefined);
 
+void Kotlin_konan_internal_GC_suspend(KRef);
+void Kotlin_konan_internal_GC_resume(KRef);
+
 #ifdef __cplusplus
 }
 #endif

runtime/src/main/kotlin/konan/internal/RuntimeUtils.kt

@@ -17,6 +17,7 @@
 package konan.internal
 
 import kotlin.internal.getProgressionLastElement
+import kotlin.text.toUtf8Array
 
 @ExportForCppRuntime
 fun ThrowNullPointerException(): Nothing {
@@ -101,3 +102,21 @@ fun getProgressionLast(start: Char, end: Char, step: Int): Char =
 
 fun getProgressionLast(start: Int, end: Int, step: Int): Int = getProgressionLastElement(start, end, step)
 fun getProgressionLast(start: Long, end: Long, step: Long): Long = getProgressionLastElement(start, end, step)
+
+// Called by the debugger.
+@ExportForCppRuntime
+fun KonanObjectToUtf8Array(value: Any?): ByteArray {
+    val string = when (value) {
+        is Array<*> -> value.contentToString()
+        is CharArray -> value.contentToString()
+        is BooleanArray -> value.contentToString()
+        is ByteArray -> value.contentToString()
+        is ShortArray -> value.contentToString()
+        is IntArray -> value.contentToString()
+        is LongArray -> value.contentToString()
+        is FloatArray -> value.contentToString()
+        is DoubleArray -> value.contentToString()
+        else -> value.toString()
+    }
+    return toUtf8Array(string, 0, string.length)
+}

runtime/src/main/kotlin/konan/internal/Util.kt

@@ -1,36 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package konan.internal
-
-import kotlin.text.toUtf8Array
-
-// Called by debugger.
-@ExportForCppRuntime
-fun KonanObjectToUtf8Array(value: Any?): ByteArray {
-    val string = when (value) {
-        is Array<*> -> value.contentToString()
-        is CharArray -> value.contentToString()
-        is BooleanArray -> value.contentToString()
-        is ByteArray -> value.contentToString()
-        is ShortArray -> value.contentToString()
-        is IntArray -> value.contentToString()
-        is LongArray -> value.contentToString()
-        is FloatArray -> value.contentToString()
-        is DoubleArray -> value.contentToString()
-        else -> value.toString()
-    }
-    return toUtf8Array(string, 0, string.length)
-}