Group slots together. (#166)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -18,6 +18,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
     var constructedClass: ClassDescriptor? = null
     val vars = VariableManager(this)
     var returnSlot: LLVMValueRef? = null
+    var slotsPhi: LLVMValueRef? = null
+    var slotCount = 0
 
     fun prologue(descriptor: FunctionDescriptor) {
         prologue(llvmFunction(descriptor),
@@ -41,17 +43,34 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
         entryBb = LLVMAppendBasicBlock(function, "entry")
         epilogueBb = LLVMAppendBasicBlock(function, "epilogue")
         positionAtEnd(entryBb!!)
+        slotsPhi = phi(kObjHeaderPtrPtr)
+        slotCount = 0
     }
 
     fun epilogue() {
         appendingTo(prologueBb!!) {
+            val slots = if (slotCount > 0)
+                LLVMBuildArrayAlloca(builder, kObjHeaderPtr, Int32(slotCount).llvm, "")!!
+            else
+                kNullObjHeaderPtrPtr
+            if (slotCount > 0) {
+                // Zero-init slots.
+                val slotsMem = bitcast(kInt8Ptr, slots)
+                val pointerSize = LLVMABISizeOfType(llvmTargetData, kObjHeaderPtr).toInt()
+                val alignment = LLVMABIAlignmentOfType(llvmTargetData, kObjHeaderPtr)
+                call(context.llvm.memsetFunction,
+                        listOf(slotsMem, Int8(0).llvm,
+                                Int32(slotCount * pointerSize).llvm, Int32(alignment).llvm,
+                                Int1(0).llvm))
+            }
+            addPhiIncoming(slotsPhi!!, prologueBb!! to slots)
             br(entryBb!!)
         }
 
         appendingTo(epilogueBb!!) {
             when {
                returnType == voidType -> {
-                   vars.releaseVars()
+                   releaseVars()
                    assert(returnSlot == null)
                    LLVMBuildRetVoid(builder)
                }
@@ -61,7 +80,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
                     if (returnSlot != null) {
                         updateLocalRef(returnPhi, returnSlot!!)
                     }
-                    vars.releaseVars()
+                    releaseVars()
                     LLVMBuildRet(builder, returnPhi)
                }
                // Do nothing, all paths throw.
@@ -72,6 +91,14 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
         returns.clear()
         vars.clear()
         returnSlot = null
+        slotsPhi = null
+    }
+
+    fun releaseVars() {
+        if (slotCount > 0) {
+            call(context.llvm.releaseLocalRefsFunction,
+                    listOf(slotsPhi, Int32(slotCount).llvm))
+        }
     }
 
     private var prologueBb: LLVMBasicBlockRef? = null
@@ -104,11 +131,11 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
     fun intToPtr(imm: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, imm, DestTy, Name)!!
 
     fun alloca(type: LLVMTypeRef?, name: String = ""): LLVMValueRef {
+        if (isObjectType(type!!)) {
+            return gep(slotsPhi!!, Int32(slotCount++).llvm)
+        }
         appendingTo(prologueBb!!) {
-            val result = LLVMBuildAlloca(builder, type, name)!!
+            return LLVMBuildAlloca(builder, type, name)!!
-            if (isObjectType(type!!))
-                LLVMBuildStore(builder, kNullObjHeaderPtr, result)
-            return result
         }
     }
     fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildLoad(builder, value, name)!!
@@ -132,6 +159,12 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
         }
     }
 
+    fun gep(base: LLVMValueRef, index: LLVMValueRef): LLVMValueRef {
+        memScoped {
+            val args = allocArrayOf(index)
+            return LLVMBuildGEP(builder, base, args[0].ptr, 1, "")!!
+        }
+    }
 
     // Only use ignoreOld, when sure that memory is freshly inited and have no value.
     fun updateLocalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -205,6 +205,14 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
         return LLVMAddFunction(llvmModule, name, functionType)!!
     }
 
+    private fun importMemset() : LLVMValueRef {
+        memScoped {
+            val parameterTypes = allocArrayOf(int8TypePtr, int8Type, int32Type, int32Type, int1Type)
+            val functionType = LLVMFunctionType(LLVMVoidType(), parameterTypes[0].ptr, 5, 0)
+            return LLVMAddFunction(llvmModule, "llvm.memset.p0i8.i32", functionType)!!
+        }
+    }
+
     val staticData = StaticData(context)
 
     val runtimeFile = context.config.configuration.get(KonanConfigKeys.RUNTIME_FILE)!!
@@ -226,6 +234,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val setGlobalRefFunction = importRtFunction("SetGlobalRef")
     val updateLocalRefFunction = importRtFunction("UpdateLocalRef")
     val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef")
+    val releaseLocalRefsFunction = importRtFunction("ReleaseLocalRefs")
     val setArrayFunction = importRtFunction("Kotlin_Array_set")
     val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl")
     val lookupFieldOffset = importRtFunction("LookupFieldOffset")
@@ -234,6 +243,7 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val checkInstanceFunction = importRtFunction("CheckInstance")
     val throwExceptionFunction = importRtFunction("ThrowException")
     val appendToInitalizersTail = importRtFunction("AppendToInitializersTail")
+    val memsetFunction = importMemset()
     val usedFunctions = mutableListOf<LLVMValueRef>()
     val staticInitializers = mutableListOf<LLVMValueRef>()
     val fileInitializers = mutableListOf<IrElement>()

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -68,6 +68,10 @@ internal open class Struct(val type: LLVMTypeRef?, val elements: List<ConstValue
     }
 }
 
+internal class Int1(val value: Byte) : ConstValue {
+    override val llvm = LLVMConstInt(LLVMInt1Type(), value.toLong(), 1)!!
+}
+
 internal class Int8(val value: Byte) : ConstValue {
     override val llvm = LLVMConstInt(LLVMInt8Type(), value.toLong(), 1)!!
 }
@@ -96,6 +100,7 @@ internal fun constValue(value: LLVMValueRef) = object : ConstValue {
     override val llvm = value
 }
 
+internal val int1Type = LLVMInt1Type()!!
 internal val int8Type = LLVMInt8Type()!!
 internal val int32Type = LLVMInt32Type()!!
 internal val int8TypePtr = pointerType(int8Type)
@@ -129,6 +134,8 @@ internal val kImmInt32One  = Int32(1).llvm
 internal val kImmInt64One  = Int64(1).llvm
 internal val ContextUtils.kNullObjHeaderPtr: LLVMValueRef
     get() = LLVMConstNull(this.kObjHeaderPtr)!!
+internal val ContextUtils.kNullObjHeaderPtrPtr: LLVMValueRef
+    get() = LLVMConstNull(this.kObjHeaderPtrPtr)!!
 
 // Nothing type has no values, but we do generate unreachable code and thus need some fake value:
 internal val ContextUtils.kNothingFakeValue: LLVMValueRef

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt

@@ -52,14 +52,6 @@ internal class VariableManager(val codegen: CodeGenerator) {
         descriptors.clear()
     }
 
-    fun releaseVars() {
-        // This function is called by codegen to cleanup local references when leaving frame.
-        for (variable in variables) {
-            if (variable.isRefSlot())
-                codegen.updateLocalRef(codegen.kNullObjHeaderPtr, variable.address())
-        }
-    }
-
     fun createVariable(scoped: Pair<VariableDescriptor, CodeContext>, value: LLVMValueRef? = null) : Int {
         // Note that we always create slot for object references for memory management.
         val descriptor = scoped.first

runtime/src/main/cpp/Memory.cpp

@@ -55,17 +55,16 @@ void FreeContainer(ContainerHeader* header) {
   ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
   const TypeInfo* typeInfo = obj->type_info();
 
+  // We use *local* versions as no other threads could see dead objects.
   for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
     ObjHeader** location = reinterpret_cast<ObjHeader**>(
         reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
-    UpdateGlobalRef(location, nullptr);
+    UpdateLocalRef(location, nullptr);
   }
   // Object arrays are *special*.
   if (typeInfo == theArrayTypeInfo) {
     ArrayHeader* array = obj->array();
-    for (int index = 0; index < array->count_; index++) {
+    ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_);
-       UpdateGlobalRef(ArrayAddressOfElementAt(array, index), nullptr);
-    }
   }
 
   // And release underlying memory.
@@ -152,9 +151,7 @@ inline void ReleaseRef(const ObjHeader* object) {
   Release(object->container());
 }
 
-#ifdef __cplusplus
 extern "C" {
-#endif
 
 void InitMemory() {
   RuntimeAssert(offsetof(ArrayHeader, type_info_)
@@ -321,6 +318,50 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) {
 #endif
 }
 
-#ifdef __cplusplus
+void ReleaseLocalRefs(ObjHeader** start, int count) {
-}
+#if TRACE_MEMORY
+  printf("ReleaseLocalRefs %p .. %p\n", start, start + count);
 #endif
+  ObjHeader** current = start;
+  while (count-- > 0) {
+    ObjHeader* object = *current;
+    if (object != nullptr) {
+      ReleaseRef(object);
+      // Just for sanity, optional.
+      *current = nullptr;
+    }
+    current++;
+  }
+}
+
+void ReleaseGlobalRefs(ObjHeader** start, int count) {
+#if TRACE_MEMORY
+  printf("ReleaseGlobalRefs %p .. %p\n", start, start + count);
+#endif
+#if CONCURRENT
+  ObjHeader** current = start;
+  while (count-- > 0) {
+    ObjHeader* object = *current;
+    if (object != nullptr) {
+      bool written = __sync_bool_compare_and_swap(
+          current, object, nullptr);
+      if (written)
+        ReleaseRef(object);
+    }
+    current++;
+  }
+#else
+  ObjHeader** current = start;
+  while (count-- > 0) {
+    ObjHeader* object = *current;
+    if (object != nullptr) {
+      ReleaseRef(object);
+      // Usually required.
+      *current = nullptr;
+    }
+    current++;
+  }
+#endif
+}
+
+} // extern "C"

runtime/src/main/cpp/Memory.h

@@ -351,6 +351,9 @@ void SetGlobalRef(ObjHeader** location, const ObjHeader* object);
 void UpdateLocalRef(ObjHeader** location, const ObjHeader* object);
 // Update potentially globally visible location.
 void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object);
+// Optimization: release all references in range.
+void ReleaseLocalRefs(ObjHeader** start, int count);
+void ReleaseGlobalRefs(ObjHeader** start, int count);
 
 #ifdef __cplusplus
 }