Frame-local allocations. (#193)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -20,6 +20,7 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
     var returnSlot: LLVMValueRef? = null
     var slotsPhi: LLVMValueRef? = null
     var slotCount = 0
+    var localAllocs = 0
 
     fun prologue(descriptor: FunctionDescriptor) {
         prologue(llvmFunction(descriptor),
@@ -45,16 +46,18 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
         cleanupLandingpad = LLVMAppendBasicBlock(function, "cleanup_landingpad")!!
         positionAtEnd(entryBb!!)
         slotsPhi = phi(kObjHeaderPtrPtr)
-        slotCount = 0
+        // First slot can be assigned to keep pointer to frame local arena.
+        slotCount = 1
+        localAllocs = 0
     }
 
     fun epilogue() {
         appendingTo(prologueBb!!) {
-            val slots = if (slotCount > 0)
+            val slots = if (needSlots)
                 LLVMBuildArrayAlloca(builder, kObjHeaderPtr, Int32(slotCount).llvm, "")!!
             else
                 kNullObjHeaderPtrPtr
-            if (slotCount > 0) {
+            if (needSlots) {
                 // Zero-init slots.
                 val slotsMem = bitcast(kInt8Ptr, slots)
                 val pointerSize = LLVMABISizeOfType(llvmTargetData, kObjHeaderPtr).toInt()
@@ -102,9 +105,14 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
         slotsPhi = null
     }
 
-    fun releaseVars() {
+    private val needSlots: Boolean
-        if (slotCount > 0) {
+        get() {
-            call(context.llvm.releaseLocalRefsFunction,
+            return slotCount > 1 || localAllocs > 0
+        }
+
+    private fun releaseVars() {
+        if (needSlots) {
+            call(context.llvm.leaveFrameFunction,
                     listOf(slotsPhi!!, Int32(slotCount).llvm))
         }
     }
@@ -148,6 +156,36 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
             return LLVMBuildAlloca(builder, type, name)!!
         }
     }
+
+
+    // Return object slot (ab)used for arena matching given allocation.
+    private fun arenaSlot() : LLVMValueRef {
+        return gep(slotsPhi!!, Int32(0).llvm)
+    }
+
+    fun allocInstance(typeInfo: LLVMValueRef, hint: Int) : LLVMValueRef {
+        if (hint == SCOPE_FRAME) {
+            val aux = arenaSlot()
+            localAllocs++
+            return call(context.llvm.arenaAllocInstanceFunction, listOf(typeInfo, aux))
+        } else {
+            val slot = vars.createAnonymousSlot()
+            return call(context.llvm.allocInstanceFunction, listOf(typeInfo, slot))
+        }
+    }
+
+    fun allocArray(
+          typeInfo: LLVMValueRef, hint: Int, count: LLVMValueRef) : LLVMValueRef {
+        if (hint == SCOPE_FRAME) {
+            val aux = arenaSlot()
+            localAllocs++
+            return call(context.llvm.arenaAllocArrayFunction, listOf(typeInfo, count, aux))
+        } else {
+            val slot = vars.createAnonymousSlot()
+            return call(context.llvm.allocArrayFunction, listOf(typeInfo, count, slot))
+        }
+    }
+
     fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef {
         val result = LLVMBuildLoad(builder, value, name)!!
         // Use loadSlot() API for that.

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -24,6 +24,12 @@ import org.jetbrains.kotlin.resolve.descriptorUtil.getSuperClassNotAny
 import org.jetbrains.kotlin.types.KotlinType
 import org.jetbrains.kotlin.types.TypeUtils
 
+// Different scopes/lifetimes of an object, computed by escape analysis.
+const val SCOPE_FRAME = 0
+const val SCOPE_GLOBAL = 1
+const val SCOPE_ARENA = 2
+const val SCOPE_PERMANENT = 3
+
 /**
  * Provides utility methods to the implementer.
  */
@@ -263,13 +269,15 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     var globalInitIndex:Int = 0
 
     val allocInstanceFunction = importRtFunction("AllocInstance")
-    val initInstanceFunction = importRtFunction("InitInstance")
+    val arenaAllocInstanceFunction = importRtFunction("ArenaAllocInstance")
     val allocArrayFunction = importRtFunction("AllocArrayInstance")
+    val arenaAllocArrayFunction = importRtFunction("ArenaAllocArrayInstance")
+    val initInstanceFunction = importRtFunction("InitInstance")
     val setLocalRefFunction = importRtFunction("SetLocalRef")
     val setGlobalRefFunction = importRtFunction("SetGlobalRef")
     val updateLocalRefFunction = importRtFunction("UpdateLocalRef")
     val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef")
-    val releaseLocalRefsFunction = importRtFunction("ReleaseLocalRefs")
+    val leaveFrameFunction = importRtFunction("LeaveFrame")
     val setArrayFunction = importRtFunction("Kotlin_Array_set")
     val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl")
     val lookupFieldOffset = importRtFunction("LookupFieldOffset")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/EscapeAnalysis.kt

@@ -0,0 +1,36 @@
+package org.jetbrains.kotlin.backend.konan.llvm
+
+import org.jetbrains.kotlin.ir.IrElement
+import org.jetbrains.kotlin.ir.declarations.IrModuleFragment
+import org.jetbrains.kotlin.ir.expressions.IrCall
+import org.jetbrains.kotlin.ir.visitors.IrElementVisitorVoid
+import org.jetbrains.kotlin.ir.visitors.acceptChildrenVoid
+import org.jetbrains.kotlin.ir.visitors.acceptVoid
+
+// Analysis we're implementing here is as following.
+// We build graph with the following nodes:
+//    * allocation set, keeping tuple of [local, ctor call, owner function], AS
+//    * local store set, keeping pair [local, stored], LSS
+//    * field store set, keeping tuple [local, stored], FSS
+//    * global store set, [local, stored], GSS
+// Function we're trying to compute is the following:
+//   for each element of AS, could it be referred by someone, whose value is
+//   alive on return from function, where element was allocated.
+// Each element in RS is associated with few elements in AS, which it could refer to.
+// TODO: exact algorithm TBD.
+internal class EscapeAnalyzerVisitor(val allocHints: MutableMap<IrCall, Int>) : IrElementVisitorVoid {
+
+    override fun visitElement(element: IrElement) {
+        element.acceptChildrenVoid(this)
+    }
+
+    override fun visitModuleFragment(module: IrModuleFragment) {
+        module.acceptChildrenVoid(this)
+    }
+}
+
+fun prepareAllocHints(irModule: IrModuleFragment, allocHints: MutableMap<IrCall, Int>) {
+    assert(allocHints.size == 0)
+
+    irModule.acceptVoid(EscapeAnalyzerVisitor(allocHints))
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -172,6 +172,7 @@ interface CodeContext {
 internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid {
 
     val codegen = CodeGenerator(context)
+    val allocHints = mutableMapOf<IrCall, Int>()
 
     //-------------------------------------------------------------------------//
 
@@ -240,6 +241,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     override fun visitModuleFragment(module: IrModuleFragment) {
         context.log("visitModule                  : ${ir2string(module)}")
 
+        prepareAllocHints(module, allocHints)
+
         module.acceptChildrenVoid(this)
         appendLlvmUsed(context.llvm.usedFunctions)
         appendStaticInitializers(context.llvm.staticInitializers)
@@ -691,10 +694,9 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
 
         codegen.positionAtEnd(bbInit)
         val typeInfo = codegen.typeInfoValue(value.descriptor)
-        val allocHint = Int32(1).llvm
         val initFunction = value.descriptor.constructors.first { it.valueParameters.size == 0 }
         val ctor = codegen.llvmFunction(initFunction)
-        val args = listOf(objectPtr, typeInfo, allocHint, ctor)
+        val args = listOf(objectPtr, typeInfo, ctor)
         val newValue = call(context.llvm.initInstanceFunction, args)
         val bbInitResult = codegen.currentBlock
         codegen.br(bbExit)
@@ -802,9 +804,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
             codegen.plus(sum, size!!)
         }
 
-        val typeInfo = codegen.typeInfoValue(value.type)!!
+        val array = codegen.allocArray(codegen.typeInfoValue(value.type)!!, SCOPE_GLOBAL, finalLength)
-        val arrayCreationArgs = listOf(typeInfo, kImmInt32One, finalLength)
-        val array = call(context.llvm.allocArrayFunction, arrayCreationArgs)
         elements.fold(kImmZero) { sum, (exp, size, isArray) ->
             if (!isArray) {
                 call(context.llvm.setArrayFunction, listOf(array, sum, exp))
@@ -832,6 +832,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     val kStringLength = KonanPlatform.builtIns.string.getter2Descriptor(kNameLength)
     val kStringBuilderToString = kStringBuilder.signature2Descriptor(kNameToString)
 
+    //TODO: make it lowering pass.
     private fun evaluateStringConcatenation(value: IrStringConcatenation): LLVMValueRef {
         data class Element(val string: LLVMValueRef, val llvmLenght: LLVMValueRef?, val length: Int)
 
@@ -857,8 +858,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
 
         val constructor = kStringBuilder!!.constructors
                 .firstOrNull { it -> it.valueParameters.size == 1 && KotlinBuiltIns.isInt(it.valueParameters[0].type) }!!
-        val stringBuilderObj = call(context.llvm.allocInstanceFunction,
+        val stringBuilderObj = codegen.allocInstance(codegen.typeInfoValue(kStringBuilder), SCOPE_FRAME)
-                listOf(codegen.typeInfoValue(kStringBuilder.defaultType)!!, kImmOne))
+
         call(codegen.llvmFunction(constructor), listOf(stringBuilderObj, totalLength))
 
         stringsWithLengths.fold(stringBuilderObj) { sum, (string, _, _) ->
@@ -1434,11 +1435,10 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
 
     */
     private fun fieldPtrOfClass(thisPtr: LLVMValueRef, value: PropertyDescriptor): LLVMValueRef {
-        val objHeaderPtr = codegen.bitcast(codegen.kObjHeaderPtr, thisPtr)
         val typePtr = pointerType(codegen.classType(value.containingDeclaration as ClassDescriptor))
         memScoped {
             val args = allocArrayOf(kImmOne)
-            val objectPtr = LLVMBuildGEP(codegen.builder, objHeaderPtr,  args[0].ptr, 1, "")
+            val objectPtr = LLVMBuildGEP(codegen.builder, thisPtr,  args[0].ptr, 1, "")
             val typedObjPtr = codegen.bitcast(typePtr, objectPtr!!)
             val fieldPtr = LLVMBuildStructGEP(codegen.builder, typedObjPtr, codegen.indexInClass(value), "")
             return fieldPtr!!
@@ -1742,24 +1742,22 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     }
 
     //-------------------------------------------------------------------------//
+    private fun hintForCall(callee: IrCall): Int {
+        return allocHints.getOrElse(callee) { SCOPE_GLOBAL }
+    }
 
     private fun evaluateConstructorCall(callee: IrCall, args: List<LLVMValueRef>): LLVMValueRef {
         context.log("evaluateConstructorCall    : ${ir2string(callee)}")
         memScoped {
             val constructedClass = (callee.descriptor as ConstructorDescriptor).constructedClass
-            val typeInfo = codegen.typeInfoValue(constructedClass)
-            val allocHint = Int32(1).llvm
             val thisValue = if (constructedClass.isArray) {
                 assert(args.size >= 1 && args[0].type == int32Type)
-                val allocArrayInstanceArgs = listOf(typeInfo, allocHint, args[0])
+                codegen.allocArray(codegen.typeInfoValue(constructedClass), hintForCall(callee), args[0])
-                call(context.llvm.allocArrayFunction, allocArrayInstanceArgs)
             } else {
-                call(context.llvm.allocInstanceFunction, listOf(typeInfo, allocHint))
+                codegen.allocInstance(codegen.typeInfoValue(constructedClass), hintForCall(callee))
             }
-            val constructorParams: MutableList<LLVMValueRef> = mutableListOf()
+            evaluateSimpleFunctionCall(callee.descriptor as FunctionDescriptor,
-            constructorParams += thisValue
+                    listOf(thisValue) + args)
-            constructorParams += args
-            evaluateSimpleFunctionCall(callee.descriptor as FunctionDescriptor, constructorParams)
             return thisValue
         }
     }
@@ -1772,8 +1770,8 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
 
         return when (name) {
             "konan.internal.areEqualByValue" -> {
-                val arg0 = args[0]!!
+                val arg0 = args[0]
-                val arg1 = args[1]!!
+                val arg1 = args[1]
                 assert (arg0.type == arg1.type)
 
                 when (LLVMGetTypeKind(arg0.type)) {
@@ -1801,12 +1799,12 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
         val descriptor = callee.descriptor
         val ib = context.irModule!!.irBuiltins
         when (descriptor) {
-            ib.eqeqeq     -> return codegen.icmpEq(args[0]!!, args[1]!!)
+            ib.eqeqeq     -> return codegen.icmpEq(args[0], args[1])
-            ib.gt0        -> return codegen.icmpGt(args[0]!!, kImmZero)
+            ib.gt0        -> return codegen.icmpGt(args[0], kImmZero)
-            ib.gteq0      -> return codegen.icmpGe(args[0]!!, kImmZero)
+            ib.gteq0      -> return codegen.icmpGe(args[0], kImmZero)
-            ib.lt0        -> return codegen.icmpLt(args[0]!!, kImmZero)
+            ib.lt0        -> return codegen.icmpLt(args[0], kImmZero)
-            ib.lteq0      -> return codegen.icmpLe(args[0]!!, kImmZero)
+            ib.lteq0      -> return codegen.icmpLe(args[0], kImmZero)
-            ib.booleanNot -> return codegen.icmpNe(args[0]!!, kTrue)
+            ib.booleanNot -> return codegen.icmpNe(args[0], kTrue)
             else -> {
                 TODO(descriptor.name.toString())
             }
@@ -1838,7 +1836,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
         if (resultPhi != null && !isNothing)
             codegen.assignPhis(resultPhi to brResult)
         if (bbExit != null && !isNothing)
-            codegen.br(bbExit!!)
+            codegen.br(bbExit)
         if (bbNext != null)                                                          // Switch generation to next or exit.
             codegen.positionAtEnd(bbNext)
         else if (bbExit != null)
@@ -1878,7 +1876,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
             val slot = codegen.gep(vtable, Int32(index).llvm)
             codegen.load(slot)
         } else {
-            // Otherwise, call via hashtable.
+            // Otherwise, call by hash.
             // TODO: optimize by storing interface number in lower bits of 'this' pointer
             //       when passing object as an interface. This way we can use those bits as index
             //       for an additional per-interface vtable.
@@ -1914,7 +1912,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
     private fun call(function: LLVMValueRef, args: List<LLVMValueRef>): LLVMValueRef {
         if (codegen.isObjectReturn(function.type)) {
             // If function returns an object - create slot for the returned value.
-            // This allows appropriate rootset accounting by just looking on stack slots.
+            // This allows appropriate rootset accounting by just looking at the stack slots.
             val resultSlot = codegen.vars.createAnonymousSlot()
             return currentCodeContext.genCall(function, args + resultSlot)
         } else {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -115,6 +115,8 @@ internal val ContextUtils.kTypeInfo: LLVMTypeRef
     get() = LLVMGetTypeByName(context.llvmModule, "struct.TypeInfo")!!
 internal val ContextUtils.kObjHeader: LLVMTypeRef
     get() = LLVMGetTypeByName(context.llvmModule, "struct.ObjHeader")!!
+internal val ContextUtils.kContainerHeader: LLVMTypeRef
+    get() = LLVMGetTypeByName(context.llvmModule, "struct.ContainerHeader")!!
 internal val ContextUtils.kObjHeaderPtr: LLVMTypeRef
     get() = pointerType(kObjHeader)
 internal val ContextUtils.kObjHeaderPtrPtr: LLVMTypeRef
@@ -129,7 +131,7 @@ internal val kInt1         = LLVMInt1Type()!!
 internal val kBoolean      = kInt1
 internal val kInt8Ptr      = pointerType(int8Type)
 internal val kInt8PtrPtr   = pointerType(kInt8Ptr)
-internal val kNullInt8Ptr  = LLVMConstNull(kInt8Ptr)
+internal val kNullInt8Ptr  = LLVMConstNull(kInt8Ptr)!!
 internal val kImmInt32One  = Int32(1).llvm
 internal val kImmInt64One  = Int64(1).llvm
 internal val ContextUtils.kNullObjHeaderPtr: LLVMValueRef

runtime/src/launcher/cpp/launcher.cpp

@@ -8,10 +8,10 @@
 
 OBJ_GETTER(setupArgs, int argc, char** argv) {
   // The count is one less, because we skip argv[0] which is the binary name.
-  AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, argc - 1, OBJ_RESULT);
+  AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT);
   ArrayHeader* array = (*OBJ_RESULT)->array();
   for (int index = 1; index < argc; index++) {
-    AllocStringInstance(SCOPE_GLOBAL, argv[index], strlen(argv[index]),
+    AllocStringInstance(argv[index], strlen(argv[index]),
                         ArrayAddressOfElementAt(array, index - 1));
   }
   RETURN_OBJ_RESULT();

runtime/src/main/cpp/Exceptions.cpp

@@ -50,13 +50,12 @@ OBJ_GETTER0(GetCurrentStackTrace) {
   RuntimeAssert(symbols != nullptr, "Not enough memory to retrieve the stacktrace");
 
   AutoFree autoFree(symbols);
-  AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, size, OBJ_RESULT);
+  AllocArrayInstance(theArrayTypeInfo, size, OBJ_RESULT);
 
   ArrayHeader* array = (*OBJ_RESULT)->array();
   for (int index = 0; index < size; ++index) {
-    AllocStringInstance(
+    AllocStringInstance(symbols[index], strlen(symbols[index]),
-        SCOPE_GLOBAL, symbols[index], strlen(symbols[index]),
+                        ArrayAddressOfElementAt(array, index));
-        ArrayAddressOfElementAt(array, index));
   }
 
   RETURN_OBJ_RESULT();

runtime/src/main/cpp/Memory.cpp

@@ -21,11 +21,16 @@
 #define TRACE_GC_PHASES 0
 
 ContainerHeader ObjHeader::theStaticObjectsContainer = {
-  CONTAINER_TAG_NOCOUNT | CONTAINER_TAG_INCREMENT
+  CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT
 };
 
 namespace {
 
+// Granularity of arena container chunks.
+constexpr container_size_t kContainerAlignment = 1024;
+// Single object alignment.
+constexpr container_size_t kObjectAlignment = 8;
+
 #if USE_GC
 // Collection threshold default (collect after having so many elements in the
 // release candidates set).
@@ -63,11 +68,30 @@ struct MemoryState {
 
 MemoryState* memoryState = nullptr;
 
-#if USE_GC
+// TODO: use those allocators for STL containers as well.
-bool isPermanent(const ContainerHeader* header) {
+template <typename T>
-  return (header->ref_count_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NOCOUNT;
+inline T* allocMemory(container_size_t size) {
+  return reinterpret_cast<T*>(calloc(1, size));
 }
 
+inline void freeMemory(void* memory) {
+  free(memory);
+}
+
+inline bool isFreeable(const ContainerHeader* header) {
+  return (header->refCount_ & CONTAINER_TAG_MASK) < CONTAINER_TAG_PERMANENT;
+}
+
+inline bool isPermanent(const ContainerHeader* header) {
+  return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
+}
+
+inline container_size_t alignUp(container_size_t size, int alignment) {
+  return (size + alignment - 1) & ~(alignment - 1);
+}
+
+#if USE_GC
+
 // Must be vector or map 'container -> number', to keep reference counters correct.
 ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
   ContainerHeaderList result;
@@ -98,8 +122,8 @@ ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
 void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) {
   fprintf(stderr, "%s: %p (%08x): %d refs %s\n",
           prefix,
-          header, header->ref_count_, header->ref_count_ >> CONTAINER_TAG_SHIFT,
+          header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT,
-          (header->ref_count_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
+          (header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
   seen->insert(header);
   auto children = collectMutableReferred(header);
   for (auto child : children) {
@@ -117,22 +141,22 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
 }
 
 void phase1(ContainerHeader* header) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0)
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0)
     return;
-  header->ref_count_ |= CONTAINER_TAG_SEEN;
+  header->refCount_ |= CONTAINER_TAG_SEEN;
   auto containers = collectMutableReferred(header);
   for (auto container : containers) {
-      container->ref_count_ -= CONTAINER_TAG_INCREMENT;
+      container->refCount_ -= CONTAINER_TAG_INCREMENT;
       phase1(container);
   }
 }
 
 void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) == 0)
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0)
     return;
-  if ((header->ref_count_ >> CONTAINER_TAG_SHIFT) != 0)
+  if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0)
     rootset->insert(header);
-  header->ref_count_ &= ~CONTAINER_TAG_SEEN;
+  header->refCount_ &= ~CONTAINER_TAG_SEEN;
   auto containers = collectMutableReferred(header);
   for (auto container : containers) {
     phase2(container, rootset);
@@ -140,32 +164,32 @@ void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) {
 }
 
 void phase3(ContainerHeader* header) {
-  if ((header->ref_count_ & CONTAINER_TAG_SEEN) != 0) {
+  if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) {
     return;
   }
-  header->ref_count_ |= CONTAINER_TAG_SEEN;
+  header->refCount_ |= CONTAINER_TAG_SEEN;
   auto containers = collectMutableReferred(header);
   for (auto container : containers) {
-    container->ref_count_ += CONTAINER_TAG_INCREMENT;
+    container->refCount_ += CONTAINER_TAG_INCREMENT;
     phase3(container);
   }
 }
 
 void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) {
-  auto ref_count = header->ref_count_ >> CONTAINER_TAG_SHIFT;
+  auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT;
-  bool seen = (ref_count > 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) == 0) ||
+  bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) ||
-      (ref_count == 0 && (header->ref_count_ & CONTAINER_TAG_SEEN) != 0);
+      (refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0);
   if (seen) return;
 
   // Add to toRemove set.
-  if (ref_count == 0)
+  if (refCount == 0)
     toRemove->insert(header);
 
   // Update seen bit.
-  if (ref_count == 0)
+  if (refCount == 0)
-    header->ref_count_ |= CONTAINER_TAG_SEEN;
+    header->refCount_ |= CONTAINER_TAG_SEEN;
   else
-    header->ref_count_ &= ~CONTAINER_TAG_SEEN;
+    header->refCount_ &= ~CONTAINER_TAG_SEEN;
   auto containers = collectMutableReferred(header);
   for (auto container : containers) {
     phase4(container, toRemove);
@@ -174,91 +198,109 @@ void phase4(ContainerHeader* header, ContainerHeaderSet* toRemove) {
 
 #endif // USE_GC
 
+// We use first slot as place to store frame-local arena container.
+ArenaContainer* initedArena(ObjHeader** auxSlot) {
+  ObjHeader* slotValue = *auxSlot;
+  if (slotValue) return reinterpret_cast<ArenaContainer*>(slotValue);
+  ArenaContainer* arena = allocMemory<ArenaContainer>(sizeof(ArenaContainer));
+  arena->Init();
+  *auxSlot = reinterpret_cast<ObjHeader*>(arena);
+  return arena;
+}
+
 }  // namespace
 
 ContainerHeader* AllocContainer(size_t size) {
-  ContainerHeader* result = reinterpret_cast<ContainerHeader*>(calloc(1, size));
+  ContainerHeader* result = allocMemory<ContainerHeader>(size);
 #if TRACE_MEMORY
-  fprintf(stderr, ">>> alloc %d -> %p\n", (int)size, result);
+  fprintf(stderr, ">>> alloc %d -> %p\n", static_cast<int>(size), result);
-   memoryState->containers->insert(result);
+  memoryState->containers->insert(result);
 #endif
   // TODO: atomic increment in concurrent case.
   memoryState->allocCount++;
   return result;
 }
 
+// TODO: shall we do padding for alignment?
+uint32_t ObjectSize(const ObjHeader* obj) {
+  const TypeInfo* type_info = obj->type_info();
+  if (type_info->instanceSize_ < 0) {
+    // An array.
+    return ArrayDataSizeBytes(obj->array()) + sizeof(ArrayHeader);
+  } else {
+    return type_info->instanceSize_ + sizeof(ObjHeader);
+  }
+}
+
 void FreeContainer(ContainerHeader* header) {
   RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
 #if TRACE_MEMORY
-  fprintf(stderr, "<<< free %p\n", header);
+  if (isFreeable(header)) {
-  memoryState->containers->erase(header);
+    fprintf(stderr, "<<< free %p\n", header);
+    memoryState->containers->erase(header);
+  }
 #endif
-  header->ref_count_ = CONTAINER_TAG_INVALID;
+
 #if USE_GC
-  if (memoryState->toFree)
+  if (memoryState->toFree && isFreeable(header))
     memoryState->toFree->erase(header);
 #endif
   // Now let's clean all object's fields in this container.
-  // TODO: this is gross hack, relying on the fact that we now only alloc
-  // ArenaContainer and ObjectContainer, which both have single element.
   ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
-  const TypeInfo* typeInfo = obj->type_info();
 
-  // We use *local* versions as no other threads could see dead objects.
+  for (int index = 0; index < header->objectCount_; index++) {
-  for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
+    const TypeInfo* typeInfo = obj->type_info();
-    ObjHeader** location = reinterpret_cast<ObjHeader**>(
+
-        reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
+    // We use *local* versions as no other threads could see dead objects.
-    UpdateLocalRef(location, nullptr);
+    for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
-  }
+      ObjHeader** location = reinterpret_cast<ObjHeader**>(
-  // Object arrays are *special*.
+          reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
-  if (typeInfo == theArrayTypeInfo) {
+      UpdateLocalRef(location, nullptr);
-    ArrayHeader* array = obj->array();
+    }
-    ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_);
+    // Object arrays are *special*.
+    if (typeInfo == theArrayTypeInfo) {
+      ArrayHeader* array = obj->array();
+      ReleaseLocalRefs(ArrayAddressOfElementAt(array, 0), array->count_);
+    }
+    obj = reinterpret_cast<ObjHeader*>(reinterpret_cast<uintptr_t>(obj) + ObjectSize(obj));
   }
 
   // And release underlying memory.
-  // TODO: atomic decrement in concurrent case.
+  if (isFreeable(header)) {
+    // TODO: atomic decrement in concurrent case.
 #if CONCURRENT
-  #error "Atomic update of allocCount"
+    #error "Atomic update of allocCount"
 #endif
-  memoryState->allocCount--;
+    memoryState->allocCount--;
-  free(header);
+    freeMemory(header);
+  }
 }
 
 #if USE_GC
 void FreeContainerNoRef(ContainerHeader* header) {
-  RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
+  RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed");
 #if TRACE_MEMORY
   fprintf(stderr, "<<< free %p\n", header);
   memoryState->containers->erase(header);
 #endif
-  header->ref_count_ = CONTAINER_TAG_INVALID;
 #if USE_GC
   if (memoryState->toFree)
     memoryState->toFree->erase(header);
 #endif
   memoryState->allocCount--;
-  free(header);
+  freeMemory(header);
 }
 #endif
 
-ArenaContainer::ArenaContainer(uint32_t size) {
-    ArenaContainerHeader* header =
-        static_cast<ArenaContainerHeader*>(AllocContainer(size + sizeof(ArenaContainerHeader)));
-    header_ = header;
-    // header->ref_count_ is zero initialized by AllocContainer().
-    header->current_ =
-        reinterpret_cast<uint8_t*>(header_) + sizeof(ArenaContainerHeader);
-    header->end_ = header->current_ + size;
-}
-
 void ObjectContainer::Init(const TypeInfo* type_info) {
   RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
   uint32_t alloc_size =
       sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_;
   header_ = AllocContainer(alloc_size);
   if (header_) {
-     // header->ref_count_ is zero initialized by AllocContainer().
+    // One object in this container.
+    header_->objectCount_ = 1;
+     // header->refCount_ is zero initialized by AllocContainer().
     SetMeta(GetPlace(), type_info);
 #if TRACE_MEMORY
     fprintf(stderr, "object at %p\n", GetPlace());
@@ -274,7 +316,9 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
   header_ = AllocContainer(alloc_size);
   RuntimeAssert(header_ != nullptr, "Cannot alloc memory");
   if (header_) {
-    // header->ref_count_ is zero initialized by AllocContainer().
+    // One object in this container.
+    header_->objectCount_ = 1;
+    // header->refCount_ is zero initialized by AllocContainer().
     GetPlace()->count_ = elements;
     SetMeta(GetPlace()->obj(), type_info);
 #if TRACE_MEMORY
@@ -283,25 +327,73 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
   }
 }
 
-ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
+void ArenaContainer::Init() {
-  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
+  allocContainer(1024);
-  uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
+}
-  ObjHeader* result = reinterpret_cast<ObjHeader*>(Place(size));
+
-  if (!result) {
+void ArenaContainer::Deinit() {
-      return nullptr;
+  auto chunk = currentChunk_;
+  while (chunk != nullptr) {
+    auto toRemove = chunk;
+    // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set.
+    FreeContainer(chunk->asHeader());
+    chunk = chunk->next;
+    freeMemory(toRemove);
   }
-  SetMeta(result, type_info);
+}
+
+bool ArenaContainer::allocContainer(container_size_t minSize) {
+  auto size = minSize + sizeof(ContainerHeader) + sizeof(ContainerChunk);
+  size = alignUp(size, kContainerAlignment);
+  ContainerChunk* result = allocMemory<ContainerChunk>(size);
+  RuntimeAssert(result != nullptr, "Cannot alloc memory");
+  if (result == nullptr) return false;
+  result->next = currentChunk_;
+  result->asHeader()->refCount_ = (CONTAINER_TAG_STACK | CONTAINER_TAG_INCREMENT);
+  currentChunk_ = result;
+  current_ = reinterpret_cast<uint8_t*>(result->asHeader() + 1);
+  end_ = reinterpret_cast<uint8_t*>(result) + size;
+  return true;
+}
+
+void* ArenaContainer::place(container_size_t size) {
+  size = alignUp(size, kObjectAlignment);
+  // Fast path.
+  if (current_ + size < end_) {
+    void* result = current_;
+    current_ += size;
+    return result;
+  }
+  if (!allocContainer(size)) {
+    return nullptr;
+  }
+  void* result = current_;
+  current_ += size;
+  RuntimeAssert(current_ <= end_, "Must not overflow");
   return result;
 }
 
-ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, int count) {
+ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
+  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
+  uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
+  ObjHeader* result = reinterpret_cast<ObjHeader*>(place(size));
+  if (!result) {
+      return nullptr;
+  }
+  currentChunk_->asHeader()->objectCount_++;
+  setMeta(result, type_info);
+  return result;
+}
+
+ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t count) {
   RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
-  uint32_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count;
+  container_size_t size = sizeof(ArrayHeader) - type_info->instanceSize_ * count;
-  ArrayHeader* result = reinterpret_cast<ArrayHeader*>(Place(size));
+  ArrayHeader* result = reinterpret_cast<ArrayHeader*>(place(size));
   if (!result) {
     return nullptr;
   }
-  SetMeta(result->obj(), type_info);
+  currentChunk_->asHeader()->objectCount_++;
+  setMeta(result->obj(), type_info);
   result->count_ = count;
   return result;
 }
@@ -384,6 +476,8 @@ void DeinitMemory() {
 
 #if USE_GC
   GarbageCollect();
+  delete memoryState->toFree;
+  memoryState->toFree = nullptr;
 #endif // USE_GC
 
   if (memoryState->allocCount > 0) {
@@ -399,20 +493,28 @@ void DeinitMemory() {
   memoryState = nullptr;
 }
 
-// Now we ignore all placement hints and always allocate heap space for new object.
+ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) {
-OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint) {
+  RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
+  return initedArena(auxSlot)->PlaceObject(type_info);
+}
+
+OBJ_GETTER(AllocInstance, const TypeInfo* type_info) {
   RuntimeAssert(type_info->instanceSize_ >= 0, "must be an object");
   RETURN_OBJ(ObjectContainer(type_info).GetPlace());
 }
 
-OBJ_GETTER(AllocArrayInstance,
+ObjHeader* ArenaAllocArrayInstance(
-           const TypeInfo* type_info, PlacementHint hint, uint32_t elements) {
+    const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) {
+  RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
+  return initedArena(auxSlot)->PlaceArray(type_info, elements)->obj();
+}
+
+OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) {
   RuntimeAssert(type_info->instanceSize_ < 0, "must be an array");
   RETURN_OBJ(ArrayContainer(type_info, elements).GetPlace()->obj());
 }
 
-OBJ_GETTER(AllocStringInstance,
+OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) {
-  PlacementHint hint, const char* data, uint32_t length) {
   ArrayHeader* array = ArrayContainer(theStringTypeInfo, length).GetPlace();
   memcpy(
       ByteArrayAddressOfElementAt(array, 0),
@@ -422,8 +524,7 @@ OBJ_GETTER(AllocStringInstance,
 }
 
 OBJ_GETTER(InitInstance,
-    ObjHeader** location, const TypeInfo* type_info, PlacementHint hint,
+    ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) {
-    void (*ctor)(ObjHeader*)) {
   ObjHeader* sentinel = reinterpret_cast<ObjHeader*>(1);
   ObjHeader* value;
   // Wait until other initializers.
@@ -438,7 +539,7 @@ OBJ_GETTER(InitInstance,
     RETURN_OBJ(value);
   }
 
-  AllocInstance(type_info, hint, OBJ_RESULT);
+  AllocInstance(type_info, OBJ_RESULT);
   ObjHeader* object = *OBJ_RESULT;
   UpdateGlobalRef(location, object);
   try {
@@ -523,6 +624,15 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) {
 #endif
 }
 
+void LeaveFrame(ObjHeader** start, int count) {
+  ReleaseLocalRefs(start + 1, count - 1);
+  if (*start != nullptr) {
+    auto arena = initedArena(start);
+    arena->Deinit();
+    freeMemory(arena);
+  }
+}
+
 void ReleaseLocalRefs(ObjHeader** start, int count) {
 #if TRACE_MEMORY
   fprintf(stderr, "ReleaseLocalRefs %p .. %p\n", start, start + count);
@@ -623,7 +733,7 @@ void GarbageCollect() {
   memoryState->toFree->clear();
 
   for (auto header : toRemove) {
-    RuntimeAssert((header->ref_count_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen");
+    RuntimeAssert((header->refCount_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen");
     FreeContainerNoRef(header);
   }
 

runtime/src/main/cpp/Memory.h

@@ -10,7 +10,7 @@ typedef enum {
   SCOPE_FRAME = 0,
   // Allocation is generic global allocation.
   SCOPE_GLOBAL = 1,
-  // Allocation shall take place in current arena.
+  // Allocation shall take place in current stack arena.
   SCOPE_ARENA = 2,
   // Allocation is permanent.
   SCOPE_PERMANENT = 3
@@ -20,15 +20,16 @@ typedef enum {
 typedef enum {
   // Container is normal thread local container.
   CONTAINER_TAG_NORMAL = 0,
-  // Container shall not be refcounted (const data, frame locals).
+   // Container shall be atomically refcounted.
-  CONTAINER_TAG_NOCOUNT = 1,
+  CONTAINER_TAG_SHARED = 1,
-  // Container shall be atomically refcounted.
+  // Those container tags shall not be refcounted.
-  CONTAINER_TAG_SHARED = 2,
+  // Permanent object, cannot refer to non-permanent objects, so no need to cleanup those.
-  // Container is no longer valid.
+  CONTAINER_TAG_PERMANENT = 2,
-  CONTAINER_TAG_INVALID = 3,
+  // Stack objects, no need to free, children cleanup still shall be there.
+  CONTAINER_TAG_STACK = 3,
   // Container was seen during GC.
   CONTAINER_TAG_SEEN = 4,
-  // Shift to get actual counter..
+  // Shift to get actual counter.
   CONTAINER_TAG_SHIFT = 3,
   // Actual value to increment/decrement conatiner by. Tag is in lower bits.
   CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
@@ -36,14 +37,17 @@ typedef enum {
   CONTAINER_TAG_MASK = ((CONTAINER_TAG_INCREMENT >> 1) - 1)
 } ContainerTag;
 
-// Could be made 64-bit for large memory configs.
 typedef uint32_t container_offset_t;
+typedef uint32_t container_size_t;
+
 
 // Header of all container objects. Contains reference counter.
 struct ContainerHeader {
   // Reference counter of container. Uses two lower bits of counter for
   // container type (for polymorphism in ::Release()).
-  volatile uint32_t ref_count_;
+  volatile uint32_t refCount_;
+  // Number of objects in the container.
+  uint32_t objectCount_;
 };
 
 struct ArrayHeader;
@@ -116,37 +120,25 @@ struct ArrayHeader {
   uint32_t count_;
 };
 
-struct ArenaContainerHeader : public ContainerHeader {
-  // Current allocation limit.
-  uint8_t* current_;
-  // Allocation end. Maybe consider having chunked backing storage
-  // at cost of smarter ::Release() polymorphic on container type.
-  uint8_t* end_;
-};
-
 inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
   // Instance size is negative.
   return -obj->type_info()->instanceSize_ * obj->count_;
 }
 
-void FreeContainer(ContainerHeader* header);
-
 // TODO: those two operations can be implemented by translator when storing
 // reference to an object.
 inline void AddRef(ContainerHeader* header) {
   // Looking at container type we may want to skip AddRef() totally
   // (non-escaping stack objects, constant objects).
-  switch (header->ref_count_ & CONTAINER_TAG_MASK) {
+  switch (header->refCount_ & CONTAINER_TAG_MASK) {
-    case CONTAINER_TAG_NORMAL:
+    case CONTAINER_TAG_STACK:
-      header->ref_count_ += CONTAINER_TAG_INCREMENT;
+    case CONTAINER_TAG_PERMANENT:
       break;
-    case CONTAINER_TAG_NOCOUNT:
+    case CONTAINER_TAG_NORMAL:
+      header->refCount_ += CONTAINER_TAG_INCREMENT;
       break;
     case CONTAINER_TAG_SHARED:
-      __sync_fetch_and_add(&header->ref_count_, CONTAINER_TAG_INCREMENT);
+      __sync_fetch_and_add(&header->refCount_, CONTAINER_TAG_INCREMENT);
-      break;
-    case CONTAINER_TAG_INVALID:
-      RuntimeAssert(false, "trying to addref invalid container");
       break;
     default:
       RuntimeAssert(false, "unknown container type");
@@ -154,19 +146,22 @@ inline void AddRef(ContainerHeader* header) {
   }
 }
 
-// Release returns 'true' iff container cannot be part of cycle (either NOCOUNT
+void FreeContainer(ContainerHeader* header);
+
+// Release() returns 'true' iff container cannot be part of cycle (either NOCOUNT
 // object or container was fully released and will be collected).
 inline bool Release(ContainerHeader* header) {
-  switch (header->ref_count_ & CONTAINER_TAG_MASK) {
+  switch (header->refCount_ & CONTAINER_TAG_MASK) {
+      case CONTAINER_TAG_PERMANENT:
+      case CONTAINER_TAG_STACK:
+        // permanent/stack containers aren't loop candidates.
+        return true;
     case CONTAINER_TAG_NORMAL:
-      if ((header->ref_count_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) {
+      if ((header->refCount_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) {
         FreeContainer(header);
         return true;
       }
       break;
-    case CONTAINER_TAG_NOCOUNT:
-      // NOCOUNT containers aren't loop candidate.
-      return true;
     // Note that shared containers have potentially subtle race, if object holds a
     // reference to another object, stored in shorter living container. In this
     // case there's unlikely, but possible case, where one mutator takes reference,
@@ -182,14 +177,11 @@ inline bool Release(ContainerHeader* header) {
     //    probability even further.
     case CONTAINER_TAG_SHARED:
       if (__sync_sub_and_fetch(
-              &header->ref_count_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) {
+              &header->refCount_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) {
         FreeContainer(header);
         return true;
       }
       break;
-    case CONTAINER_TAG_INVALID:
-      RuntimeAssert(false, "trying to release invalid container");
-      break;
     default:
       RuntimeAssert(false, "unknown container type");
       break;
@@ -236,9 +228,9 @@ class ObjectContainer : public Container {
 
   // Object container shalln't have any dtor, as it's being freed by
   // ::Release().
+
   ObjHeader* GetPlace() const {
-    return reinterpret_cast<ObjHeader*>(
+    return reinterpret_cast<ObjHeader*>(header_ + 1);
-        reinterpret_cast<uint8_t*>(header_) + sizeof(ContainerHeader));
   }
 
  private:
@@ -253,41 +245,23 @@ class ArrayContainer : public Container {
   }
 
   // Array container shalln't have any dtor, as it's being freed by ::Release().
+
   ArrayHeader* GetPlace() const {
-    return reinterpret_cast<ArrayHeader*>(
+    return reinterpret_cast<ArrayHeader*>(header_ + 1);
-        reinterpret_cast<uint8_t*>(header_) + sizeof(ContainerHeader));
   }
 
  private:
   void Init(const TypeInfo* type_info, uint32_t elements);
 };
 
-
 // Class representing arena-style placement container.
-// Container is used for reference counting,
+// Container is used for reference counting, and it is assumed that objects
-// and it is assumed that objects with related placement will share container. Only
+// with related placement will share container. Only
 // whole container can be freed, individual objects are not taken into account.
-class ArenaContainer : public Container {
+class ArenaContainer {
  public:
-  explicit ArenaContainer(uint32_t size);
+  void Init();
-
+  void Deinit();
-  ~ArenaContainer() {
-    if (header_) {
-      RuntimeAssert(header_->ref_count_ == 0, "Non-zero refcount");
-      Dispose();
-    }
-  }
-
-  // Allocation function.
-  void* Place(int size) {
-    ArenaContainerHeader* header = reinterpret_cast<ArenaContainerHeader*>(header_);
-    if (header->current_ + size > header->end_) {
-      return nullptr;
-    }
-    void* result = header->current_;
-    header->current_ += size;
-    return result;
-  }
 
   // Place individual object in this container.
   ObjHeader* PlaceObject(const TypeInfo* type_info);
@@ -295,15 +269,28 @@ class ArenaContainer : public Container {
   // Places an array of certain type in this container. Note that array_type_info
   // is type info for an array, not for an individual element. Also note that exactly
   // same operation could be used to place strings.
-  ArrayHeader* PlaceArray(const TypeInfo* array_type_info, int count);
+  ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count);
 
-  // Dispose whole container ignoring non-zero refcount. Use with care.
+ private:
-  void Dispose() {
+  struct ContainerChunk {
-    if (header_) {
+    ContainerChunk* next;
-      FreeContainer(header_);
+    // Then we have ContainerHeader here.
-      header_ = nullptr;
+    ContainerHeader* asHeader() {
+      return reinterpret_cast<ContainerHeader*>(this + 1);
     }
+  };
+
+  void* place(container_size_t size);
+  bool allocContainer(container_size_t minSize);
+  void setMeta(ObjHeader* obj, const TypeInfo* type_info) {
+    obj->container_offset_negative_ =
+        reinterpret_cast<uintptr_t>(obj) - reinterpret_cast<uintptr_t>(currentChunk_->asHeader());
+    obj->set_type_info(type_info);
+    RuntimeAssert(obj->container() == currentChunk_->asHeader(), "Placement must match");
   }
+  ContainerChunk* currentChunk_;
+  uint8_t* current_;
+  uint8_t* end_;
 };
 
 #ifdef __cplusplus
@@ -321,14 +308,26 @@ extern "C" {
 void InitMemory();
 void DeinitMemory();
 
-OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint);
+//
-OBJ_GETTER(AllocArrayInstance,
+// Object allocation.
-           const TypeInfo* type_info, PlacementHint hint, uint32_t elements);
+//
-OBJ_GETTER(AllocStringInstance,
+// Allocation can happen in either GLOBAL, FRAME or ARENA scope. Depending on that,
-           PlacementHint hint, const char* data, uint32_t length);
+// Alloc* or ArenaAlloc* is called. Regular alloc means allocation happens in the heap,
+// and each object gets its individual container. Otherwise, allocator uses aux slot in
+// an implementation-defined manner, current behavior is to keep arena pointer there.
+// Arena containers are not reference counted, and is explicitly freed when leaving
+// its owner frame.
+// Escape analysis algorithm is the provider of information for decision on exact aux slot
+// selection, and comes from upper bound esteemation of object lifetime.
+//
+ObjHeader* ArenaAllocInstance(const TypeInfo* type_info, ObjHeader** auxSlot) RUNTIME_NOTHROW;
+OBJ_GETTER(AllocInstance, const TypeInfo* type_info) RUNTIME_NOTHROW;
+ObjHeader* ArenaAllocArrayInstance(
+    const TypeInfo* type_info, uint32_t elements, ObjHeader** auxSlot) RUNTIME_NOTHROW;
+OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, uint32_t elements) RUNTIME_NOTHROW;
+OBJ_GETTER(AllocStringInstance, const char* data, uint32_t length) RUNTIME_NOTHROW;
 OBJ_GETTER(InitInstance,
-           ObjHeader** location, const TypeInfo* type_info, PlacementHint hint,
+           ObjHeader** location, const TypeInfo* type_info, void (*ctor)(ObjHeader*)) RUNTIME_NOTHROW;
-           void (*ctor)(ObjHeader*));
 
 //
 // Object reference management.
@@ -363,6 +362,8 @@ void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTH
 // Optimization: release all references in range.
 void ReleaseLocalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW;
 void ReleaseGlobalRefs(ObjHeader** start, int count) RUNTIME_NOTHROW;
+// Called on frame leave, if it has object slots.
+void LeaveFrame(ObjHeader** start, int count) RUNTIME_NOTHROW;
 // Collect garbage, which cannot be found by reference counting (cycles).
 void GarbageCollect() RUNTIME_NOTHROW;
 

runtime/src/main/cpp/Operator.cpp

@@ -5,7 +5,7 @@
 
 namespace {
 
-inline template<typename R, typename Ta, typename Tb> R div(Ta a, Tb b) {
+template<typename R, typename Ta, typename Tb> R div(Ta a, Tb b) {
     if (__builtin_expect(b == 0, false)) {
         ThrowArithmeticException();
     }

runtime/src/main/kotlin/kotlin/Arrays.kt

@@ -2,6 +2,8 @@ package kotlin
 
 import kotlin.collections.*
 
+// TODO: make all iterator() methods inline.
+
 /**
  * An array of bytes.
  * @constructor Creates a new array of the specified [size], with all elements initialized to zero.
@@ -27,7 +29,7 @@ public final class ByteArray : Cloneable {
     external private fun getArrayLength(): Int
 
     /** Creates an iterator over the elements of the array. */
-    public operator fun iterator(): kotlin.collections.ByteIterator {
+    public operator fun iterator(): ByteIterator {
         return ByteIteratorImpl(this)
     }
 }