Memory management design and implementation. (#148)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -103,20 +103,47 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
 
     fun intToPtr(imm: LLVMValueRef?, DestTy: LLVMTypeRef, Name: String = "") = LLVMBuildIntToPtr(builder, imm, DestTy, Name)
 
-    fun alloca(type: KotlinType, name: String = ""): LLVMValueRef = alloca(getLLVMType(type), name)
     fun alloca(type: LLVMTypeRef?, name: String = ""): LLVMValueRef {
         appendingTo(prologueBb!!) {
-            return LLVMBuildAlloca(builder, type, name)!!
+            val result = LLVMBuildAlloca(builder, type, name)!!
+            if (isObjectType(type!!))
+                LLVMBuildStore(builder, kNullObjHeaderPtr, result)
+            return result
         }
     }
     fun load(value: LLVMValueRef, name: String = ""): LLVMValueRef = LLVMBuildLoad(builder, value, name)!!
-    fun store(value: LLVMValueRef, ptr: LLVMValueRef): LLVMValueRef = LLVMBuildStore(builder, value, ptr)!!
+    fun store(value: LLVMValueRef, ptr: LLVMValueRef) {
+        // Use updateRef() or storeAny() API for that.
+        assert(!isObjectRef(value))
+        LLVMBuildStore(builder, value, ptr)
+    }
+    fun storeAnyLocal(value: LLVMValueRef, ptr: LLVMValueRef) {
+        if (isObjectRef(value)) {
+            updateLocalRef(value, ptr)
+        } else {
+            LLVMBuildStore(builder, value, ptr)
+        }
+    }
+    fun storeAnyGlobal(value: LLVMValueRef, ptr: LLVMValueRef) {
+        if (isObjectRef(value)) {
+            updateGlobalRef(value, ptr)
+        } else {
+            LLVMBuildStore(builder, value, ptr)
+        }
+    }
 
+
+    // Only use ignoreOld, when sure that memory is freshly inited and have no value.
     fun updateLocalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) {
         call(if (ignoreOld) context.llvm.setLocalRefFunction else context.llvm.updateLocalRefFunction,
                 listOf(address, value))
     }
 
+    fun updateGlobalRef(value: LLVMValueRef, address: LLVMValueRef, ignoreOld: Boolean = false) {
+        call(if (ignoreOld) context.llvm.setGlobalRefFunction else context.llvm.updateGlobalRefFunction,
+                listOf(address, value))
+    }
+
     fun isConst(value: LLVMValueRef): Boolean = (LLVMIsConstant(value) == 1)
 
     //-------------------------------------------------------------------------//

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -222,7 +222,9 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     val initInstanceFunction = importRtFunction("InitInstance")
     val allocArrayFunction = importRtFunction("AllocArrayInstance")
     val setLocalRefFunction = importRtFunction("SetLocalRef")
+    val setGlobalRefFunction = importRtFunction("SetGlobalRef")
     val updateLocalRefFunction = importRtFunction("UpdateLocalRef")
+    val updateGlobalRefFunction = importRtFunction("UpdateGlobalRef")
     val setArrayFunction = importRtFunction("Kotlin_Array_set")
     val copyImplArrayFunction = importRtFunction("Kotlin_Array_copyImpl")
     val lookupFieldOffset = importRtFunction("LookupFieldOffset")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -204,7 +204,6 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
      * Convenient [InnerScope] implementation that is bound to the [currentCodeContext].
      */
     private abstract inner class InnerScopeImpl : InnerScope(currentCodeContext)
-
     /**
      * Executes [block] with [codeContext] substituted as [currentCodeContext].
      */
@@ -250,7 +249,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
                 val descriptor = irField.descriptor
                 val initialization = evaluateExpression(irField.initializer)
                 val globalPtr = LLVMGetNamedGlobal(context.llvmModule, descriptor.symbolName)
-                codegen.store(initialization!!, globalPtr!!)
+                codegen.storeAnyGlobal(initialization!!, globalPtr!!)
             }
             codegen.ret(null)
         }
@@ -412,7 +411,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
                         fieldDeclaration.initializer?.let {
                             val value = evaluateExpression(it)!!
                             val fieldPtr = fieldPtrOfClass(thisPtr, fieldDescriptor)
-                            codegen.store(value, fieldPtr)
+                            codegen.storeAnyGlobal(value, fieldPtr)
                         }
                     }
 
@@ -1450,11 +1449,11 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
         val valueToAssign = evaluateExpression(value.value)!!
         if (value.descriptor.dispatchReceiverParameter != null) {
             val thisPtr = instanceFieldAccessReceiver(value)
-            codegen.store(valueToAssign, fieldPtrOfClass(thisPtr, value.descriptor))
+            codegen.storeAnyGlobal(valueToAssign, fieldPtrOfClass(thisPtr, value.descriptor))
         }
         else {
             val globalValue = LLVMGetNamedGlobal(context.llvmModule, value.descriptor.symbolName)
-            codegen.store(valueToAssign, globalValue!!)
+            codegen.storeAnyGlobal(valueToAssign, globalValue!!)
         }
 
         return null
@@ -1818,6 +1817,7 @@ internal class CodeGeneratorVisitor(val context: Context) : IrElementVisitorVoid
                                  branch: IrBranch, bbNext: LLVMBasicBlockRef?, bbExit: LLVMBasicBlockRef?) {
         val neitherUnitNorNothing = !isNothing && !isUnit                            // If branches doesn't end with 'return' either result hasn't got 'unit' type.
         val branchResult = branch.result
+        // TODO: use phis here!
         if (isUnconditional(branch)) {                                               // It is the "else" clause.
             val brResult = evaluateExpression(branchResult)                          // Generate clause body.
             if (neitherUnitNorNothing)                                               // If nor unit neither result ends with return

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/LlvmUtils.kt

@@ -142,10 +142,6 @@ internal fun structType(types: List<LLVMTypeRef>): LLVMTypeRef = memScoped {
     LLVMStructType(allocArrayOf(types)[0].ptr, types.size, 0)!!
 }
 
-internal fun ContextUtils.isObjectType(type: LLVMTypeRef) : Boolean {
-    return type == kObjHeaderPtr || type == kArrayHeaderPtr
-}
-
 internal fun ContextUtils.getLlvmFunctionType(function: FunctionDescriptor): LLVMTypeRef {
     val returnType = if (function is ConstructorDescriptor) voidType else getLLVMType(function.returnType!!)
     val paramTypes = ArrayList(function.allValueParameters.map { getLLVMType(it.type) })
@@ -168,6 +164,14 @@ internal fun ContextUtils.isObjectReturn(functionType: LLVMTypeRef) : Boolean {
     return isObjectType(returnType)
 }
 
+internal fun ContextUtils.isObjectRef(value: LLVMValueRef): Boolean {
+    return isObjectType(value.type)
+}
+
+internal fun ContextUtils.isObjectType(type: LLVMTypeRef): Boolean {
+    return type == kObjHeaderPtr || type == kArrayHeaderPtr
+}
+
 /**
  * Reads [size] bytes contained in this array.
  */
@@ -208,6 +212,7 @@ internal fun functionType(returnType: LLVMTypeRef, isVarArg: Boolean = false, va
             LLVMFunctionType(returnType, paramTypesPtr, paramTypes.size, if (isVarArg) 1 else 0)!!
         }
 
+
 fun llvm2string(value: LLVMValueRef?): String {
   if (value == null) return "<null>"
   return LLVMPrintValueToString(value)!!.asCString().toString()

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RTTIGenerator.kt

@@ -160,7 +160,7 @@ internal class RTTIGenerator(override val context: Context) : ContextUtils {
 
         val refFieldIndices = classDesc.fields.mapIndexedNotNull { index, field ->
             val type = field.returnType!!
-            if (!KotlinBuiltIns.isPrimitiveType(type)) {
+            if (isObjectType(getLLVMType(type))) {
                 index
             } else {
                 null

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/VariableManager.kt

@@ -18,7 +18,7 @@ internal class VariableManager(val codegen: CodeGenerator) {
             return codegen.load(address)
         }
         override fun store(value: LLVMValueRef) {
-            codegen.store(value, address)
+            codegen.storeAnyLocal(value, address)
         }
         override fun address() : LLVMValueRef {
             return this.address
@@ -54,10 +54,13 @@ internal class VariableManager(val codegen: CodeGenerator) {
 
     fun releaseVars() {
         // This function is called by codegen to cleanup local references when leaving frame.
+        for (variable in variables) {
+            if (variable.isRefSlot())
+                codegen.updateLocalRef(codegen.kNullObjHeaderPtr, variable.address())
+        }
     }
 
     fun createVariable(scoped: Pair<VariableDescriptor, CodeContext>, value: LLVMValueRef? = null) : Int {
-        // TODO: fix, due to the bug in frontend, we shall always create stack slot for variable now.
         // Note that we always create slot for object references for memory management.
         val descriptor = scoped.first
         if (descriptor.isVar() || codegen.isObjectType(codegen.getLLVMType(descriptor.type)) || true) {
@@ -74,7 +77,7 @@ internal class VariableManager(val codegen: CodeGenerator) {
         val type = codegen.getLLVMType(descriptor.type)
         val slot = codegen.alloca(type, descriptor.name.asString())
         if (value != null)
-            codegen.store(value, slot)
+            codegen.storeAnyLocal(value, slot)
         variables.add(SlotRecord(slot, codegen.isObjectType(type)))
         descriptors[scoped] = index
         return index
@@ -95,7 +98,7 @@ internal class VariableManager(val codegen: CodeGenerator) {
         val index = variables.size
         val slot = codegen.alloca(type)
         if (value != null)
-            codegen.store(value, slot)
+            codegen.storeAnyLocal(value, slot)
         variables.add(SlotRecord(slot, codegen.isObjectType(type)))
         return index
     }

backend.native/tests/runtime/collections/array_list1.kt

@@ -10,12 +10,12 @@ fun assertFalse(cond: Boolean) {
 
 fun assertEquals(value1: Any?, value2: Any?) {
     if (value1 != value2)
-        println("FAIL")
+        throw Error("FAIL " + value1 + " " + value2)
 }
 
 fun assertEquals(value1: Int, value2: Int) {
     if (value1 != value2)
-        println("FAIL")
+        throw Error("FAIL " + value1 + " " + value2)
 }
 
 fun testBasic() {

backend.native/tests/runtime/memory/basic0.kt

@@ -0,0 +1,10 @@
+class A {
+    var field: B? = null
+}
+
+class B(var field: Int)
+
+fun main(args : Array<String>) {
+    val a = A()
+    a.field = B(2)
+}

runtime/src/launcher/cpp/launcher.cpp

@@ -1,4 +1,5 @@
 #include <string.h>
+
 #include "Memory.h"
 #include "Natives.h"
 #include "Types.h"
@@ -9,16 +10,15 @@ OBJ_GETTER(setupArgs, int argc, char** argv) {
   // The count is one less, because we skip argv[0] which is the binary name.
   AllocArrayInstance(theArrayTypeInfo, SCOPE_GLOBAL, argc - 1, OBJ_RESULT);
   ArrayHeader* array = (*OBJ_RESULT)->array();
-  for (int index = 0; index < argc - 1; index++) {
-    AllocStringInstance(SCOPE_GLOBAL, argv[index + 1], strlen(argv[index + 1]),
-                        ArrayAddressOfElementAt(array, index));
+  for (int index = 1; index < argc; index++) {
+    AllocStringInstance(SCOPE_GLOBAL, argv[index], strlen(argv[index]),
+                        ArrayAddressOfElementAt(array, index - 1));
   }
   RETURN_OBJ_RESULT();
 }
 
 //--- main --------------------------------------------------------------------//
-
-extern "C" void Konan_start(ObjHeader* );
+extern "C" void Konan_start(const ObjHeader* );
 
 int main(int argc, char** argv) {
 
@@ -31,6 +31,8 @@ int main(int argc, char** argv) {
       Konan_start(args.obj());
     }
 
+    DeinitMemory();
+
     // Yes, we have to follow Java convention and return zero.
     return 0;
 }

runtime/src/main/cpp/Arrays.cpp

@@ -32,10 +32,10 @@ OBJ_GETTER(Kotlin_Array_clone, KConstRef thiz) {
   const ArrayHeader* array = thiz->array();
   ArrayHeader* result = ArrayContainer(
       array->type_info(), array->count_).GetPlace();
-  memcpy(
-      ArrayAddressOfElementAt(result, 0),
-      ArrayAddressOfElementAt(array, 0),
-      ArrayDataSizeBytes(array));
+  for (int index = 0; index < array->count_; index++) {
+    SetGlobalRef(
+         ArrayAddressOfElementAt(result, index), *ArrayAddressOfElementAt(array, index));
+  }
   RETURN_OBJ(result->obj());
 }
 
@@ -49,9 +49,8 @@ void Kotlin_Array_fillImpl(KRef thiz, KInt fromIndex, KInt toIndex, KRef value)
   if (fromIndex < 0 || toIndex < fromIndex || toIndex > array->count_) {
     ThrowArrayIndexOutOfBoundsException();
   }
-  // TODO: refcounting!
   for (KInt index = fromIndex; index < toIndex; ++index) {
-    *ArrayAddressOfElementAt(array, index) = value;
+    UpdateGlobalRef(ArrayAddressOfElementAt(array, index), value);
   }
 }
 
@@ -59,13 +58,21 @@ void Kotlin_Array_copyImpl(KConstRef thiz, KInt fromIndex,
                            KRef destination, KInt toIndex, KInt count) {
   const ArrayHeader* array = thiz->array();
   ArrayHeader* destinationArray = destination->array();
-   if (fromIndex < 0 || fromIndex + count > array->count_ ||
-       toIndex < 0 || toIndex + count > destinationArray->count_) {
-        ThrowArrayIndexOutOfBoundsException();
+  if (fromIndex < 0 || fromIndex + count > array->count_ ||
+      toIndex < 0 || toIndex + count > destinationArray->count_) {
+    ThrowArrayIndexOutOfBoundsException();
+  }
+  if (fromIndex >= toIndex) {
+    for (int index = 0; index < count; index++) {
+      UpdateGlobalRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
+                      *ArrayAddressOfElementAt(array, fromIndex + index));
     }
-    // TODO: refcounting!
-    memmove(ArrayAddressOfElementAt(destinationArray, toIndex),
-        ArrayAddressOfElementAt(array, fromIndex), count * sizeof(KRef));
+  } else {
+    for (int index = count - 1; index >= 0; index--) {
+      UpdateGlobalRef(ArrayAddressOfElementAt(destinationArray, toIndex + index),
+                      *ArrayAddressOfElementAt(array, fromIndex + index));
+    }
+  }
 }
 
 // Arrays.kt

runtime/src/main/cpp/Memory.cpp

@@ -1,23 +1,82 @@
-#include <string.h>
 #include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
 
 #include <cstddef> // for offsetof
+#include <set> // only for memory tracing.
+#include <vector>
 
 #include "Assert.h"
 #include "Exceptions.h"
 #include "Memory.h"
 #include "Natives.h"
 
-void FreeObject(ContainerHeader* header) {
+// Define to 1 to see all memory operations.
+#define TRACE_MEMORY 0
+// Define to 1 to use in multithreaded environment.
+#define CONCURRENT 0
+
+namespace {
+
+// Current number of allocated containers.
+int allocCount = 0;
+#if TRACE_MEMORY
+// List of all global objects addresses.
+std::vector<KRef*>* globalObjects = nullptr;
+// Set of all containers.
+std::set<ContainerHeader*>* containers = nullptr;
+#endif
+
+}  // namespace
+
+ContainerHeader* AllocContainer(size_t size) {
+  ContainerHeader* result = reinterpret_cast<ContainerHeader*>(calloc(1, size));
+#if TRACE_MEMORY
+  printf(">>> alloc %d -> %p\n", (int)size, result);
+  containers->insert(result);
+#endif
+  // TODO: atomic increment in concurrent case.
+  allocCount++;
+  return result;
+}
+
+void FreeContainer(ContainerHeader* header) {
+#if TRACE_MEMORY
+  printf("<<< free %p\n", header);
+  containers->erase(header);
+#endif
   header->ref_count_ = CONTAINER_TAG_INVALID;
+
+  // Now let's clean all object's fields in this container.
+  // TODO: this is gross hack, relying on the fact that we now only alloc
+  // ArenaContainer and ObjectContainer, which both have single element.
+  ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
+  const TypeInfo* typeInfo = obj->type_info();
+
+  for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
+    ObjHeader** location = reinterpret_cast<ObjHeader**>(
+        reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
+    UpdateGlobalRef(location, nullptr);
+  }
+  // Object arrays are *special*.
+  if (typeInfo == theArrayTypeInfo) {
+    ArrayHeader* array = obj->array();
+    for (int index = 0; index < array->count_; index++) {
+       UpdateGlobalRef(ArrayAddressOfElementAt(array, index), nullptr);
+    }
+  }
+
+  // And release underlying memory.
+  // TODO: atomic decrement in concurrent case.
+  allocCount--;
   free(header);
 }
 
 ArenaContainer::ArenaContainer(uint32_t size) {
-    ArenaContainerHeader* header = reinterpret_cast<ArenaContainerHeader*>(
-        calloc(size + sizeof(ArenaContainerHeader), 1));
+    ArenaContainerHeader* header =
+        static_cast<ArenaContainerHeader*>(AllocContainer(size + sizeof(ArenaContainerHeader)));
     header_ = header;
-    header->ref_count_ = CONTAINER_TAG_INCREMENT;
+    // header->ref_count_ is zero initialized by AllocContainer().
     header->current_ =
         reinterpret_cast<uint8_t*>(header_) + sizeof(ArenaContainerHeader);
     header->end_ = header->current_ + size;
@@ -25,12 +84,15 @@ ArenaContainer::ArenaContainer(uint32_t size) {
 
 void ObjectContainer::Init(const TypeInfo* type_info) {
   RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
-   uint32_t alloc_size =
+  uint32_t alloc_size =
       sizeof(ContainerHeader) + sizeof(ObjHeader) + type_info->instanceSize_;
-  header_ = reinterpret_cast<ContainerHeader*>(calloc(alloc_size, 1));
+  header_ = AllocContainer(alloc_size);
   if (header_) {
-    header_->ref_count_ = CONTAINER_TAG_INCREMENT;
+     // header->ref_count_ is zero initialized by AllocContainer().
     SetMeta(GetPlace(), type_info);
+#if TRACE_MEMORY
+    printf("object at %p\n", GetPlace());
+#endif
   }
 }
 
@@ -39,12 +101,15 @@ void ArrayContainer::Init(const TypeInfo* type_info, uint32_t elements) {
   uint32_t alloc_size =
       sizeof(ContainerHeader) + sizeof(ArrayHeader) -
       type_info->instanceSize_ * elements;
-  header_ = reinterpret_cast<ContainerHeader*>(calloc(alloc_size, 1));
+  header_ = AllocContainer(alloc_size);
   RuntimeAssert(header_ != nullptr, "Cannot alloc memory");
   if (header_) {
-    header_->ref_count_ = CONTAINER_TAG_INCREMENT;
+    // header->ref_count_ is zero initialized by AllocContainer().
     GetPlace()->count_ = elements;
     SetMeta(GetPlace()->obj(), type_info);
+#if TRACE_MEMORY
+    printf("array at %p\n", GetPlace());
+#endif
   }
 }
 
@@ -71,6 +136,20 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, int count) {
   return result;
 }
 
+inline void AddRef(const ObjHeader* object) {
+#if TRACE_MEMORY
+  printf("AddRef on %p in %p\n", object, object->container());
+#endif
+  AddRef(object->container());
+}
+
+inline void ReleaseRef(const ObjHeader* object) {
+#if TRACE_MEMORY
+  printf("ReleaseRef on %p in %p\n", object, object->container());
+#endif
+  Release(object->container());
+}
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -85,6 +164,35 @@ void InitMemory() {
                 offsetof(ObjHeader  , container_offset_negative_),
                 "Layout mismatch");
   // TODO: initialize heap here.
+  allocCount = 0;
+#if TRACE_MEMORY
+  globalObjects = new std::vector<KRef*>();
+  containers = new std::set<ContainerHeader*>();
+#endif
+}
+
+void DeinitMemory() {
+#if TRACE_MEMORY
+  // Free all global objects, to ensure no memory leaks happens.
+  for (auto location: *globalObjects) {
+    printf("Release global in *%p: %p\n", location, *location);
+    UpdateGlobalRef(location, nullptr);
+  }
+  delete globalObjects;
+  globalObjects = nullptr;
+#endif
+
+  if (allocCount > 0) {
+#if TRACE_MEMORY
+    // TODO: move out of TRACE_MEMORY, once exceptions free memory.
+    printf("*** Memory leaks, leaked %d containers ***\n", allocCount);
+    for (auto container: *containers) {
+      printf("Unfreed container %p, count = %d\n", container, container->ref_count_);
+    }
+    delete containers;
+    containers = nullptr;
+#endif
+  }
 }
 
 // Now we ignore all placement hints and always allocate heap space for new object.
@@ -115,6 +223,7 @@ OBJ_GETTER(InitInstance,
   ObjHeader* sentinel = reinterpret_cast<ObjHeader*>(1);
   ObjHeader* value;
   // Wait until other initializers.
+  // TODO: check CONCURRENT!
   while ((value = __sync_val_compare_and_swap(
              location, nullptr, sentinel)) == sentinel) {
     // TODO: consider yielding.
@@ -126,28 +235,88 @@ OBJ_GETTER(InitInstance,
   }
 
   AllocInstance(type_info, hint, OBJ_RESULT);
+  ObjHeader* object = *OBJ_RESULT;
   try {
-    ctor(*OBJ_RESULT);
-    bool ok = __sync_bool_compare_and_swap(location, sentinel, *OBJ_RESULT);
-    RuntimeAssert(ok, "CAS must succeed");
+    ctor(object);
+    UpdateGlobalRef(location, object);
+#if CONCURRENT
+    // TODO: locking or smth lock-free in MT case?
+#endif
+#if TRACE_MEMORY
+    globalObjects->push_back(location);
+#endif
     RETURN_OBJ_RESULT();
   } catch (...) {
-    __sync_val_compare_and_swap(location, sentinel, nullptr);
+    UpdateLocalRef(OBJ_RESULT, nullptr);
+    UpdateGlobalRef(location, nullptr);
     RETURN_OBJ(nullptr);
   }
 }
 
-// Just stubs.
 void SetLocalRef(ObjHeader** location, const ObjHeader* object) {
+#if TRACE_MEMORY
+  printf("SetLocalRef *%p: %p\n", location, object);
+#endif
   *const_cast<const ObjHeader**>(location) = object;
+  if (object != nullptr) {
+    AddRef(object);
+  }
+}
+
+void SetGlobalRef(ObjHeader** location, const ObjHeader* object) {
+#if TRACE_MEMORY
+  printf("SetGlobalRef *%p: %p\n", location, object);
+#endif
+  *const_cast<const ObjHeader**>(location) = object;
+   if (object != nullptr) {
+      AddRef(object);
+   }
+#if CONCURRENT
+   // TODO: memory fence here.
+#endif
 }
 
 void UpdateLocalRef(ObjHeader** location, const ObjHeader* object) {
-  *const_cast<const ObjHeader**>(location) = object;
+  ObjHeader* old = *location;
+#if TRACE_MEMORY
+  printf("UpdateLocalRef *%p: %p -> %p\n", location, old, object);
+#endif
+  if (old != object) {
+    *const_cast<const ObjHeader**>(location) = object;
+    if (old > reinterpret_cast<ObjHeader*>(1)) {
+      ReleaseRef(old);
+    }
+    if (object != nullptr) {
+      AddRef(object);
+    }
+  }
 }
 
 void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object) {
-  *const_cast<const ObjHeader**>(location) = object;
+#if CONCURRENT
+  ObjHeader* old = *location;
+#if TRACE_MEMORY
+  printf("UpdateGlobalRef *%p: %p -> %p\n", location, old, object);
+#endif
+  if (old != object) {
+    if (object != nullptr) {
+      AddRef(object);
+    }
+    bool written = __sync_bool_compare_and_swap(
+        location, old, const_cast<ObjHeader*>(object));
+    if (written) {
+      if (old > reinterpret_cast<ObjHeader*>(1)) {
+        ReleaseRef(old);
+      }
+    } else {
+      if (object != nullptr) {
+        ReleaseRef(object);
+      }
+    }
+  }
+#else
+  UpdateLocalRef(location, object);
+#endif
 }
 
 #ifdef __cplusplus

runtime/src/main/cpp/Memory.h

@@ -10,7 +10,9 @@ typedef enum {
   // Allocation is generic global allocation.
   SCOPE_GLOBAL = 1,
   // Allocation shall take place in current arena.
-  SCOPE_ARENA = 2
+  SCOPE_ARENA = 2,
+  // Allocation is permanent.
+  SCOPE_PERMANENT = 3
 } PlacementHint;
 
 // Must fit in two bits.
@@ -116,6 +118,8 @@ inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
   return -obj->type_info()->instanceSize_ * obj->count_;
 }
 
+void FreeContainer(ContainerHeader* header);
+
 // Those two operations are implemented by translator when storing references
 // to objects.
 inline void AddRef(ContainerHeader* header) {
@@ -136,13 +140,11 @@ inline void AddRef(ContainerHeader* header) {
   }
 }
 
-void FreeObject(ContainerHeader* header);
-
 inline void Release(ContainerHeader* header) {
   switch (header->ref_count_ & CONTAINER_TAG_MASK) {
     case CONTAINER_TAG_NORMAL:
       if ((header->ref_count_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) {
-        FreeObject(header);
+        FreeContainer(header);
       }
       break;
     case CONTAINER_TAG_NOCOUNT:
@@ -163,7 +165,7 @@ inline void Release(ContainerHeader* header) {
     case CONTAINER_TAG_SHARED:
       if (__sync_sub_and_fetch(
               &header->ref_count_, CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_SHARED) {
-        FreeObject(header);
+        FreeContainer(header);
       }
       break;
     case CONTAINER_TAG_INVALID:
@@ -182,6 +184,7 @@ class Container {
     obj->container_offset_negative_ =
         reinterpret_cast<uintptr_t>(obj) - reinterpret_cast<uintptr_t>(header_);
     obj->set_type_info(type_info);
+    RuntimeAssert(obj->container() == header_, "Placement must match");
   }
 
  public:
@@ -207,7 +210,8 @@ class ObjectContainer : public Container {
     Init(type_info);
   }
 
-  // Object container shalln't have any dtor, as it's being freed by ::Release().
+  // Object container shalln't have any dtor, as it's being freed by
+  // ::Release().
   ObjHeader* GetPlace() const {
     return reinterpret_cast<ObjHeader*>(
         reinterpret_cast<uint8_t*>(header_) + sizeof(ContainerHeader));
@@ -272,7 +276,7 @@ class ArenaContainer : public Container {
   // Dispose whole container ignoring non-zero refcount. Use with care.
   void Dispose() {
     if (header_) {
-      FreeObject(header_);
+      FreeContainer(header_);
       header_ = nullptr;
     }
   }
@@ -285,12 +289,14 @@ extern "C" {
 #define OBJ_RESULT __result__
 #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT)
 #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT)
-#define RETURN_OBJ(value) UpdateLocalRef(OBJ_RESULT, value); return value;
+#define RETURN_OBJ(value) { ObjHeader* obj = value; UpdateLocalRef(OBJ_RESULT, obj); return obj; }
 #define RETURN_OBJ_RESULT() return *OBJ_RESULT;
 #define RETURN_RESULT_OF0(name) name(OBJ_RESULT); return *OBJ_RESULT;
 #define RETURN_RESULT_OF(name, ...) name(__VA_ARGS__, OBJ_RESULT); return *OBJ_RESULT;
 
 void InitMemory();
+void DeinitMemory();
+
 OBJ_GETTER(AllocInstance, const TypeInfo* type_info, PlacementHint hint);
 OBJ_GETTER(AllocArrayInstance,
            const TypeInfo* type_info, PlacementHint hint, uint32_t elements);
@@ -309,6 +315,37 @@ void UpdateLocalRef(ObjHeader** location, const ObjHeader* object);
 // Update potentially globally visible location.
 void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object);
 
+//
+// Object reference management.
+//
+// Reference management scheme we use assumes significant degree of flexibility, so that
+// one could implement either pure reference counting scheme, or tracing collector without
+// much ado.
+// Most important primitive is UpdateRef() API, which modifies location to use new
+// object reference. In pure reference counted scheme it will check old value,
+// decrement reference, increment counter on the new value, and store it into the field.
+// In tracing collector-like scheme, only field updates counts, and all other operations are
+// essentially no-ops.
+//
+// On codegeneration phase we adopt following approaches:
+//  - every stack frame has several slots, holding object references (allRefs)
+//  - those are known by compiler (and shall be grouped together)
+//  - it keeps all locally allocated objects in such slot
+//  - all local variables keeping an object also allocate a slot
+//  - most manipulations on objects happens in SSA variables and do no affect slots
+//  - exception handlers knowns slot locations for every function, and can update references
+//    in intermediate frames when throwing
+//
+
+// Sets locally visible location.
+void SetLocalRef(ObjHeader** location, const ObjHeader* object);
+// Sets potentially globally visible location.
+void SetGlobalRef(ObjHeader** location, const ObjHeader* object);
+// Update locally visible location.
+void UpdateLocalRef(ObjHeader** location, const ObjHeader* object);
+// Update potentially globally visible location.
+void UpdateGlobalRef(ObjHeader** location, const ObjHeader* object);
+
 #ifdef __cplusplus
 }
 #endif

runtime/src/main/cpp/Types.cpp

@@ -24,6 +24,11 @@ KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info) {
   return obj_type_info != nullptr;
 }
 
+KBoolean IsArray(KConstRef obj) {
+  RuntimeAssert(obj != nullptr, "Object must not be null");
+  return obj->type_info()->instanceSize_ < 0;
+}
+
 void CheckInstance(const ObjHeader* obj, const TypeInfo* type_info) {
   if (IsInstance(obj, type_info)) {
     return;

runtime/src/main/cpp/Types.h

@@ -38,6 +38,7 @@ extern const TypeInfo* theThrowableTypeInfo;
 
 KBoolean IsInstance(const ObjHeader* obj, const TypeInfo* type_info);
 void CheckCast(const ObjHeader* obj, const TypeInfo* type_info);
+KBoolean IsArray(KConstRef obj);
 
 typedef void (*Initializer)();
 struct InitNode {