Rework memory manager to make stack slot update faster (#2813)

Interop/JsRuntime/src/main/kotlin/jsinterop.kt

@@ -27,15 +27,15 @@ typealias Pointer = Int
 /**
  * @Retain annotation is required to preserve functions from internalization and DCE.
  */
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_allocateArena")
 external public fun allocateArena(): Arena
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_freeArena")
 external public fun freeArena(arena: Arena)
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_pushIntToArena")
 external public fun pushIntToArena(arena: Arena, value: Int)
 
@@ -49,15 +49,15 @@ fun doubleUpper(value: Double): Int =
 fun doubleLower(value: Double): Int =
     (value.toBits() and 0x00000000ffffffff) .toInt()
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("ReturnSlot_getDouble")
 external public fun ReturnSlot_getDouble(): Double
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Kotlin_String_utf16pointer")
 external public fun stringPointer(message: String): Pointer
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Kotlin_String_utf16length")
 external public fun stringLengthBytes(message: String): Int
 
@@ -68,7 +68,7 @@ fun <R> wrapFunction(func: KtFunction<R>): Int {
     return ptr.toInt() // TODO: LP64 unsafe.
 }
 
-@Retain
+@RetainForTarget("wasm")
 @ExportForCppRuntime("Konan_js_runLambda")
 fun runLambda(pointer: Int, argumentsArena: Arena, argumentsArenaSize: Int): Int {
     val arguments = arrayListOf<JsValue>()
@@ -104,19 +104,19 @@ open class JsArray(arena: Arena, index: Object): JsValue(arena, index) {
         get() = this.getInt("length")
 }
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_getInt")
 external public fun getInt(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int;
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_getProperty")
 external public fun Konan_js_getProperty(arena: Arena, obj: Object, propertyPtr: Pointer, propertyLen: Int): Int;
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_setFunction")
 external public fun setFunction(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int , function: Int)
 
-@Retain
+@RetainForTarget("wasm")
 @SymbolName("Konan_js_setString")
 external public fun setString(arena: Arena, obj: Object, propertyName: Pointer, propertyLength: Int, stringPtr: Pointer, stringLength: Int )
 
@@ -140,4 +140,4 @@ fun JsValue.setter(property: String, string: String) {
 object ArenaManager {
     val globalArena = allocateArena()
     var currentArena = globalArena
-}
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/CAdapterGenerator.kt

@@ -850,11 +850,14 @@ internal class CAdapterGenerator(
         |
         |extern "C" {
         |void UpdateHeapRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
+        |void UpdateStackRef(KObjHeader**, const KObjHeader*) RUNTIME_NOTHROW;
         |KObjHeader* AllocInstance(const KTypeInfo*, KObjHeader**) RUNTIME_NOTHROW;
         |KObjHeader* DerefStablePointer(void*, KObjHeader**) RUNTIME_NOTHROW;
         |void* CreateStablePointer(KObjHeader*) RUNTIME_NOTHROW;
         |void DisposeStablePointer(void*) RUNTIME_NOTHROW;
         |int IsInstance(const KObjHeader*, const KTypeInfo*) RUNTIME_NOTHROW;
+        |void EnterFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
+        |void LeaveFrame(KObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
         |void Kotlin_initRuntimeIfNeeded();
         |
         |KObjHeader* CreateStringFromCString(const char*, KObjHeader**);
@@ -862,19 +865,32 @@ internal class CAdapterGenerator(
         |void DisposeCString(char* cstring);
         |}  // extern "C"
         |
+        |struct ${prefix}_FrameOverlay {
+        |  void* arena;
+        |  ${prefix}_FrameOverlay* previous;
+        |  ${prefix}_KInt parameters;
+        |  ${prefix}_KInt count;
+        |};
+        |
         |class KObjHolder {
         |public:
-        |  KObjHolder() : obj_(nullptr) {}
+        |  KObjHolder() : obj_(nullptr) {
+        |    EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+        |  }
         |  explicit KObjHolder(const KObjHeader* obj) : obj_(nullptr) {
-        |    UpdateHeapRef(&obj_, obj);
+        |    EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+        |    UpdateStackRef(&obj_, obj);
         |  }
         |  ~KObjHolder() {
-        |    UpdateHeapRef(&obj_, nullptr);
+        |    LeaveFrame(frame(), 0, sizeof(*this)/sizeof(void*));
         |  }
         |  KObjHeader* obj() { return obj_; }
         |  KObjHeader** slot() { return &obj_; }
         | private:
+        |  ${prefix}_FrameOverlay frame_;
         |  KObjHeader* obj_;
+        |
+        |  KObjHeader** frame() { return reinterpret_cast<KObjHeader**>(&frame_); }
         |};
         |static void DisposeStablePointerImpl(${prefix}_KNativePtr ptr) {
         |  DisposeStablePointer(ptr);

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -996,7 +996,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
 
     private val kotlinExceptionRtti: ConstPointer
         get() = constPointer(importGlobal(
-                "_ZTI9ObjHolder", // typeinfo for ObjHolder
+                "_ZTI18ExceptionObjHolder", // typeinfo for ObjHolder
                 int8TypePtr,
                 origin = context.stdlibModule.llvmSymbolOrigin
         )).bitcast(int8TypePtr)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -13,6 +13,7 @@ import org.jetbrains.kotlin.backend.konan.*
 import org.jetbrains.kotlin.backend.konan.descriptors.*
 import org.jetbrains.kotlin.backend.konan.ir.*
 import org.jetbrains.kotlin.backend.konan.llvm.coverage.*
+import org.jetbrains.kotlin.backend.konan.llvm.objcexport.is64Bit
 import org.jetbrains.kotlin.backend.konan.optimizations.*
 import org.jetbrains.kotlin.builtins.KotlinBuiltIns
 import org.jetbrains.kotlin.builtins.UnsignedType
@@ -681,7 +682,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         }
 
 
-        if (declaration.descriptor.retainAnnotation) {
+        if (declaration.descriptor.retainAnnotation(context.config.target)) {
             context.llvm.usedFunctions.add(codegen.llvmFunction(declaration))
         }
 
@@ -2190,7 +2191,7 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
         val constructedClass = functionGenerationContext.constructedClass!!
         val thisPtr = currentCodeContext.genGetValue(constructedClass.thisReceiver!!)
 
-        if (constructor.constructedClass.isExternalObjCClass()) {
+        if (constructor.constructedClass.isExternalObjCClass() || constructor.constructedClass.isAny()) {
             assert(args.isEmpty())
             return codegen.theUnitInstanceRef.llvm
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/RetainAnnotation.kt

@@ -5,13 +5,18 @@
 
 package org.jetbrains.kotlin.backend.konan.llvm
 
-import org.jetbrains.kotlin.descriptors.FunctionDescriptor
+import org.jetbrains.kotlin.backend.konan.descriptors.getStringValueOrNull
-import org.jetbrains.kotlin.descriptors.annotations.*
+import org.jetbrains.kotlin.descriptors.*
+import org.jetbrains.kotlin.konan.target.KonanTarget
 import org.jetbrains.kotlin.name.FqName
 
-private val annotationName = FqName("kotlin.native.Retain")
+private val retainAnnotationName = FqName("kotlin.native.Retain")
+private val retainForTargetAnnotationName = FqName("kotlin.native.RetainForTarget")
 
-internal val FunctionDescriptor.retainAnnotation: Boolean
+
-    get() {
+internal fun FunctionDescriptor.retainAnnotation(target: KonanTarget): Boolean {
-        return (this.annotations.findAnnotation(annotationName) != null)
+    if (this.annotations.findAnnotation(retainAnnotationName) != null) return true
-    }
+    val forTarget = this.annotations.findAnnotation(retainForTargetAnnotationName)
+    if (forTarget != null && forTarget.getStringValueOrNull("target") == target.name) return true
+    return false
+}

backend.native/tests/build.gradle

@@ -2065,7 +2065,7 @@ task extend_exception(type: RunKonanTest) {
 }
 
 task check_stacktrace_format(type: RunStandaloneKonanTest) {
-    disabled = !isAppleTarget(project)
+    disabled = true // !isAppleTarget(project)
     flags = ['-g']
     source = "runtime/exceptions/check_stacktrace_format.kt"
 }
@@ -3184,6 +3184,7 @@ task produce_dynamic(type: DynamicKonanTest) {
                 "enum100 = 100\n" +
                 "object = 42\n" +
                 "singleton = I am single\n" +
+                "mutable = foo\n" +
                 "topLevel = 777 3\n"
 }
 

backend.native/tests/interop/objc/smoke.kt

@@ -182,6 +182,8 @@ fun testWeakReference(block: () -> NSObject) {
         createAndTestWeakReference(block)
     }
 
+    kotlin.native.internal.GC.collect()
+
     assertNull(ref.get())
 }
 

backend.native/tests/produce_dynamic/simple/hello.kt

@@ -14,6 +14,10 @@ fun hello() {
 
 fun getString() = "Kotlin/Native"
 
+data class Data(var string: String)
+
+fun getMutable() = Data("foo")
+
 // Class with inheritance.
 open class Base {
     open fun foo() = println("Base.foo")

backend.native/tests/produce_dynamic/simple/main.c

@@ -16,9 +16,11 @@ int main(void) {
     T_(I) casted_impl2 = { .pinned = impl2.pinned };
     T_(Enum) enum1 = __ kotlin.root.Enum.HUNDRED.get();
     T_(Codeable) object1 = __ kotlin.root.get_an_object();
+    T_(Data) data = __ kotlin.root.getMutable();
 
     const char* string1 = __ kotlin.root.getString();
     const char* string2 = __ kotlin.root.Singleton.toString(singleton);
+    const char* string3 = __ kotlin.root.Data.get_string(data);
 
     __ kotlin.root.hello();
     __ kotlin.root.Base.foo(base);
@@ -40,6 +42,8 @@ int main(void) {
 
     printf("singleton = %s\n",  string2);
 
+    printf("mutable = %s\n",  string3);
+
     topLevelFunctionVoidFromC(42, 0);
     __ kotlin.root.topLevelFunctionVoid(42, 0);
     printf("topLevel = %d %d\n", topLevelFunctionFromC(780, 3), __ kotlin.root.topLevelFunctionFromCShort(5, 2));
@@ -49,6 +53,7 @@ int main(void) {
     __ DisposeStablePointer(singleton.pinned);
     __ DisposeString(string1);
     __ DisposeString(string2);
+    __ DisposeString(string3);
     __ DisposeStablePointer(base.pinned);
     __ DisposeStablePointer(child.pinned);
     __ DisposeStablePointer(impl1.pinned);

backend.native/tests/runtime/memory/weak0.kt

@@ -13,7 +13,6 @@ data class Data(val s: String)
 fun localWeak(): WeakReference<Data>  {
     val x = Data("Hello")
     val weak = WeakReference(x)
-
     println(weak.get())
     return weak
 }
@@ -29,10 +28,14 @@ fun multiWeak(): Array<WeakReference<Data>>  {
 
 @Test fun runTest() {
     val weak = localWeak()
+    kotlin.native.internal.GC.collect()
     val value = weak.get()
     println(value?.toString())
 
     val weaks = multiWeak()
+
+    kotlin.native.internal.GC.collect()
+
     weaks.forEach {
         it -> if (it.get()?.s != null) throw Error("not null")
     }

runtime/src/launcher/cpp/launcher.cpp

@@ -29,8 +29,9 @@ OBJ_GETTER(setupArgs, int argc, const char** argv) {
   ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, argc - 1, OBJ_RESULT);
   ArrayHeader* array = result->array();
   for (int index = 1; index < argc; index++) {
-    CreateStringFromCString(
+    ObjHolder result;
-      argv[index], ArrayAddressOfElementAt(array, index - 1));
+    CreateStringFromCString(argv[index], result.slot());
+    UpdateHeapRef(ArrayAddressOfElementAt(array, index - 1), result.obj());
   }
   return result;
 }

runtime/src/main/cpp/Atomic.cpp

@@ -208,7 +208,7 @@ OBJ_GETTER(Kotlin_AtomicReference_get, KRef thiz) {
     // rescheduled unluckily, between the moment value is read from the field and RC is incremented,
     // object may go away.
     AtomicReferenceLayout* ref = asAtomicReference(thiz);
-    RETURN_RESULT_OF(ReadRefLocked, &ref->value_, &ref->lock_);
+    RETURN_RESULT_OF(ReadHeapRefLocked, &ref->value_, &ref->lock_);
 }
 
 }  // extern "C"

runtime/src/main/cpp/Common.h

@@ -24,6 +24,7 @@
 #define RUNTIME_USED __attribute__((used))
 
 #define ALWAYS_INLINE __attribute__((always_inline))
+#define NO_INLINE __attribute__((noinline))
 
 #if KONAN_NO_THREADS
 #define THREAD_LOCAL_VARIABLE

runtime/src/main/cpp/Exceptions.cpp

@@ -156,7 +156,9 @@ OBJ_GETTER0(GetCurrentStackTrace) {
 OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
 #if OMIT_BACKTRACE
   ObjHeader* result = AllocArrayInstance(theArrayTypeInfo, 1, OBJ_RESULT);
-  CreateStringFromCString("<UNIMPLEMENTED>", ArrayAddressOfElementAt(result->array(), 0));
+  ObjHolder holder;
+  CreateStringFromCString("<UNIMPLEMENTED>", holder.slot());
+  UpdateHeapRef(ArrayAddressOfElementAt(result->array(), 0), holder.obj());
   return result;
 #else
   uint32_t size = stackTrace->array()->count_;
@@ -172,7 +174,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
     }
     char line[512];
     konan::snprintf(line, sizeof(line) - 1, "%s (%p)", symbol, (void*)(intptr_t)address);
-    CreateStringFromCString(line, ArrayAddressOfElementAt(strings->array(), index));
+    ObjHolder holder;
+    CreateStringFromCString(line, holder.slot());
+    UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj());
   }
 #else
   if (size > 0) {
@@ -195,7 +199,9 @@ OBJ_GETTER(GetStackTraceStrings, KConstRef stackTrace) {
       } else {
         result = symbol;
       }
-      CreateStringFromCString(result, ArrayAddressOfElementAt(strings->array(), index));
+      ObjHolder holder;
+      CreateStringFromCString(result, holder.slot());
+      UpdateHeapRef(ArrayAddressOfElementAt(strings->array(), index), holder.obj());
     }
   }
 #endif
@@ -210,7 +216,7 @@ void ThrowException(KRef exception) {
   PrintThrowable(exception);
   RuntimeCheck(false, "Exceptions unsupported");
 #else
-  throw ObjHolder(exception);
+  throw ExceptionObjHolder(exception);
 #endif
 }
 
@@ -276,7 +282,7 @@ static void KonanTerminateHandler() {
   } else {
     try {
       std::rethrow_exception(currentException);
-    } catch (ObjHolder& e) {
+    } catch (ExceptionObjHolder& e) {
       TerminateWithUnhandledException(e.obj());
     } catch (...) {
       // Not a Kotlin exception.

runtime/src/main/cpp/KString.cpp

@@ -1176,8 +1176,7 @@ OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endInd
     RETURN_RESULT_OF0(TheEmptyString);
   }
   KInt length = endIndex - startIndex;
-  ArrayHeader* result = AllocArrayInstance(
+  ArrayHeader* result = AllocArrayInstance(theStringTypeInfo, length, OBJ_RESULT)->array();
-    theStringTypeInfo, length, OBJ_RESULT)->array();
   memcpy(CharArrayAddressOfElementAt(result, 0),
          CharArrayAddressOfElementAt(thiz, startIndex),
          length * sizeof(KChar));
@@ -1185,14 +1184,14 @@ OBJ_GETTER(Kotlin_String_subSequence, KString thiz, KInt startIndex, KInt endInd
 }
 
 const KChar* Kotlin_String_utf16pointer(KString message) {
-    RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
+  RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
-    const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
+  const KChar* utf16 = CharArrayAddressOfElementAt(message, 0);
-    return utf16;
+  return utf16;
 }
 
 KInt Kotlin_String_utf16length(KString message) {
-    RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
+  RuntimeAssert(message->type_info() == theStringTypeInfo, "Must use a string");
-    return message->count_ * sizeof(KChar);
+  return message->count_ * sizeof(KChar);
 }
 
 

runtime/src/main/cpp/Memory.h

@@ -39,8 +39,9 @@ typedef enum {
   // Mask for container type.
   CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
 
-  // Shift to get actual object count.
+  // Shift to get actual object count, if has it.
-  CONTAINER_TAG_GC_SHIFT     = 6,
+  CONTAINER_TAG_GC_SHIFT     = 7,
+  CONTAINER_TAG_GC_MASK      = (1 << CONTAINER_TAG_GC_SHIFT) - 1,
   CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT,
   // Color mask of a container.
   CONTAINER_TAG_COLOR_SHIFT   = 3,
@@ -64,7 +65,9 @@ typedef enum {
   // Individual state bits used during GC and freezing.
   CONTAINER_TAG_GC_MARKED   = 1 << CONTAINER_TAG_COLOR_SHIFT,
   CONTAINER_TAG_GC_BUFFERED = 1 << (CONTAINER_TAG_COLOR_SHIFT + 1),
-  CONTAINER_TAG_GC_SEEN     = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2)
+  CONTAINER_TAG_GC_SEEN     = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2),
+  // If indeed has more that one object.
+  CONTAINER_TAG_GC_HAS_OBJECT_COUNT = 1 << (CONTAINER_TAG_COLOR_SHIFT + 3)
 } ContainerTag;
 
 typedef enum {
@@ -109,8 +112,8 @@ struct ContainerHeader {
     return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK;
   }
 
-  inline unsigned refCount() const {
+  inline int refCount() const {
-    return refCount_ >> CONTAINER_TAG_SHIFT;
+    return (int)refCount_ >> CONTAINER_TAG_SHIFT;
   }
 
   inline void setRefCount(unsigned refCount) {
@@ -140,20 +143,50 @@ struct ContainerHeader {
     return value >> CONTAINER_TAG_SHIFT;
   }
 
+  inline int decRefCount() {
+  #ifdef KONAN_NO_THREADS
+      int value = refCount_ -= CONTAINER_TAG_INCREMENT;
+  #else
+      int value = shareable() ?
+         __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT;
+  #endif
+      return value >> CONTAINER_TAG_SHIFT;
+  }
+
   inline unsigned tag() const {
     return refCount_ & CONTAINER_TAG_MASK;
   }
 
   inline unsigned objectCount() const {
-    return objectCount_ >> CONTAINER_TAG_GC_SHIFT;
+    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0 ?
+        (objectCount_ >> CONTAINER_TAG_GC_SHIFT) : 1;
   }
 
   inline void incObjectCount() {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0, "Must have object count");
     objectCount_ += CONTAINER_TAG_GC_INCREMENT;
   }
 
   inline void setObjectCount(int count) {
-    objectCount_ = count << CONTAINER_TAG_GC_SHIFT;
+    if (count == 1) {
+      objectCount_ &= ~CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
+    } else {
+      objectCount_ = (count << CONTAINER_TAG_GC_SHIFT) | CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
+    }
+  }
+
+  inline unsigned containerSize() const {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must be single-object");
+    return (objectCount_ >> CONTAINER_TAG_GC_SHIFT);
+  }
+
+  inline void setContainerSize(unsigned size) {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must not have object count");
+    objectCount_ = (objectCount_ & CONTAINER_TAG_GC_MASK) | (size << CONTAINER_TAG_GC_SHIFT);
+  }
+
+  inline bool hasContainerSize() {
+    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0;
   }
 
   inline unsigned color() const {
@@ -213,6 +246,7 @@ struct ContainerHeader {
     objectCount_ &= ~CONTAINER_TAG_GC_SEEN;
   }
 
+  // Following operations only work on freed container which is in finalization queue.
   // We cannot use 'this' here, as it conflicts with aliasing analysis in clang.
   inline void setNextLink(ContainerHeader* next) {
     *reinterpret_cast<ContainerHeader**>(this + 1) = next;
@@ -296,10 +330,11 @@ struct ObjHeader {
 
   ContainerHeader* container() const {
     unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
-    if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) return nullptr;
+    if ((bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) == 0)
-    return (bits & OBJECT_TAG_NONTRIVIAL_CONTAINER) != 0 ?
+      return reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(this)) - 1;
-         (reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_ :
+    if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0)
-         reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(this)) - 1;
+      return nullptr;
+    return (reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_;
   }
 
   // Unsafe cast to ArrayHeader. Use carefully!
@@ -336,6 +371,8 @@ inline bool PermanentOrFrozen(ObjHeader* obj) {
 
 // Class representing arbitrary placement container.
 class Container {
+ public:
+  ContainerHeader* header() const { return header_; }
  protected:
   // Data where everything is being stored.
   ContainerHeader* header_;
@@ -350,97 +387,10 @@ class Container {
   }
 };
 
-// Container for a single object.
-class ObjectContainer : public Container {
- public:
-  // Single instance.
-  explicit ObjectContainer(const TypeInfo* type_info) {
-    Init(type_info);
-  }
-
-  // Object container shalln't have any dtor, as it's being freed by
-  // ::Release().
-
-  ObjHeader* GetPlace() const {
-    return reinterpret_cast<ObjHeader*>(header_ + 1);
-  }
-
- private:
-  void Init(const TypeInfo* type_info);
-};
-
-
-class ArrayContainer : public Container {
- public:
-  ArrayContainer(const TypeInfo* type_info, uint32_t elements) {
-    Init(type_info, elements);
-  }
-
-  // Array container shalln't have any dtor, as it's being freed by ::Release().
-
-  ArrayHeader* GetPlace() const {
-    return reinterpret_cast<ArrayHeader*>(header_ + 1);
-  }
-
- private:
-  void Init(const TypeInfo* type_info, uint32_t elements);
-};
-
-// Class representing arena-style placement container.
-// Container is used for reference counting, and it is assumed that objects
-// with related placement will share container. Only
-// whole container can be freed, individual objects are not taken into account.
-class ArenaContainer;
-
-struct ContainerChunk {
-  ContainerChunk* next;
-  ArenaContainer* arena;
-  // Then we have ContainerHeader here.
-  ContainerHeader* asHeader() {
-    return reinterpret_cast<ContainerHeader*>(this + 1);
-  }
-};
-
-class ArenaContainer {
- public:
-  void Init();
-  void Deinit();
-
-  // Place individual object in this container.
-  ObjHeader* PlaceObject(const TypeInfo* type_info);
-
-  // Places an array of certain type in this container. Note that array_type_info
-  // is type info for an array, not for an individual element. Also note that exactly
-  // same operation could be used to place strings.
-  ArrayHeader* PlaceArray(const TypeInfo* array_type_info, container_size_t count);
-
-  ObjHeader** getSlot();
-
- private:
-  void* place(container_size_t size);
-
-  bool allocContainer(container_size_t minSize);
-
-  void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) {
-    obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
-    obj->setContainer(currentChunk_->asHeader());
-    // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC.
-  }
-
-  ContainerChunk* currentChunk_;
-  uint8_t* current_;
-  uint8_t* end_;
-  ArrayHeader* slots_;
-  uint32_t slotsCount_;
-};
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-// Bit or'ed to slot pointer, marking the fact that allocation shall happen
-// in arena pointed by the slot.
-#define ARENA_BIT 1
 #define OBJ_RESULT __result__
 #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT)
 #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT)
@@ -508,6 +458,8 @@ void WeakReferenceCounterClear(ObjHeader* counter);
 //    in intermediate frames when throwing
 //
 
+// Sets stack location.
+void SetStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Sets heap location.
 void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Zeroes heap location.
@@ -528,15 +480,11 @@ OBJ_GETTER(SwapHeapRefLocked,
 // Sets reference with taken lock.
 void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock) RUNTIME_NOTHROW;
 // Reads reference with taken lock.
-OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW;
+OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock) RUNTIME_NOTHROW;
 // Called on frame enter, if it has object slots.
 void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
 // Called on frame leave, if it has object slots.
 void LeaveFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
-// Tries to use returnSlot's arena for allocation.
-ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) RUNTIME_NOTHROW;
-// Tries to use param's arena for allocation.
-ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) RUNTIME_NOTHROW;
 // Collect garbage, which cannot be found by reference counting (cycles).
 void GarbageCollect() RUNTIME_NOTHROW;
 // Clears object subgraph references from memory subsystem, and optionally
@@ -561,27 +509,67 @@ void EnsureNeverFrozen(ObjHeader* obj);
 }
 #endif
 
+
+struct FrameOverlay {
+  void* arena;
+  FrameOverlay* previous;
+  // As they go in pair, sizeof(FrameOverlay) % sizeof(void*) == 0 is always held.
+  int32_t parameters;
+  int32_t count;
+};
+
 // Class holding reference to an object, holding object during C++ scope.
 class ObjHolder {
  public:
-   ObjHolder() : obj_(nullptr) {}
+   ObjHolder() : obj_(nullptr) {
+     EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+   }
 
    explicit ObjHolder(const ObjHeader* obj) {
-     ::SetHeapRef(&obj_, obj);
+     EnterFrame(frame(), 0, sizeof(*this)/sizeof(void*));
+     ::SetStackRef(slot(), obj);
    }
+
    ~ObjHolder() {
-     ::ZeroHeapRef(&obj_);
+     LeaveFrame(frame(), 0, sizeof(*this)/sizeof(void*));
    }
 
    ObjHeader* obj() { return obj_; }
-   const ObjHeader* obj() const { return obj_; }
-   ObjHeader** slot() { return &obj_; }
-   void clear() { ::ZeroHeapRef(&obj_); }
 
-  private:
+   const ObjHeader* obj() const { return obj_; }
+
+   ObjHeader** slot() {
+     return &obj_;
+   }
+
+   void clear() { ::ZeroStackRef(&obj_); }
+
+ private:
+   ObjHeader** frame() { return reinterpret_cast<ObjHeader**>(&frame_); }
+
+   FrameOverlay frame_;
    ObjHeader* obj_;
 };
 
+class ExceptionObjHolder {
+ public:
+   explicit ExceptionObjHolder(const ObjHeader* obj) {
+     ::SetHeapRef(&obj_, obj);
+   }
+
+   ~ExceptionObjHolder() {
+     ZeroHeapRef(&obj_);
+   }
+
+   ObjHeader* obj() { return obj_; }
+
+   const ObjHeader* obj() const { return obj_; }
+
+ private:
+   ObjHeader* obj_;
+};
+
+
 class KRefSharedHolder {
  public:
   inline ObjHeader** slotToInit() {

runtime/src/main/cpp/Worker.cpp

@@ -61,18 +61,14 @@ enum {
 
 THREAD_LOCAL_VARIABLE KInt g_currentWorkerId = 0;
 
-KNativePtr transfer(KRef object, KInt mode) {
+KNativePtr transfer(ObjHolder* holder, KInt mode) {
-  switch (mode) {
+  void* result = CreateStablePointer(holder->obj());
-    case CHECKED:
+  if (!ClearSubgraphReferences(holder->obj(), mode == CHECKED)) {
-    case UNCHECKED:
+    DisposeStablePointer(result);
-      if (!ClearSubgraphReferences(object, mode == CHECKED)) {
+    ThrowWorkerInvalidState();
-        // Release reference to the object, as it is not being managed by ObjHolder.
-        ZeroHeapRef(&object);
-        ThrowWorkerInvalidState();
-      }
-      return object;
   }
-  return nullptr;
+  holder->clear();
+  return result;
 }
 
 class Locker {
@@ -373,34 +369,33 @@ void* workerRoutine(void* argument) {
   g_currentWorkerId = worker->id();
   Kotlin_initRuntimeIfNeeded();
 
-  while (true) {
+  {
-    Job job = worker->getJob();
-    if (job.function == nullptr) {
-       // Termination request, notify the future.
-      job.future->storeResultUnlocked(nullptr, true);
-      theState()->removeWorkerUnlocked(worker->id());
-      break;
-    }
     ObjHolder argumentHolder;
-    KRef argument = AdoptStablePointer(job.argument, argumentHolder.slot());
+    ObjHolder resultHolder;
-    // Note that this is a bit hacky, as we must not auto-release resultRef,
+    while (true) {
-    // so we don't use ObjHolder.
+      Job job = worker->getJob();
-    // It is so, as ownership is transferred.
+      if (job.function == nullptr) {
-    KRef resultRef = nullptr;
+        // Termination request, notify the future.
-    KNativePtr result = nullptr;
+        job.future->storeResultUnlocked(nullptr, true);
-    bool ok = true;
+        theState()->removeWorkerUnlocked(worker->id());
-    try {
+        break;
-        job.function(argument, &resultRef);
+      }
+      KRef argument = AdoptStablePointer(job.argument, argumentHolder.slot());
+      KNativePtr result = nullptr;
+      bool ok = true;
+      try {
+        job.function(argument, resultHolder.slot());
         argumentHolder.clear();
         // Transfer the result.
-        result = transfer(resultRef, job.transferMode);
+        result = transfer(&resultHolder, job.transferMode);
-    } catch (ObjHolder& e) {
+      } catch (ExceptionObjHolder& e) {
         ok = false;
         if (worker->errorReporting())
-            ReportUnhandledException(e.obj());
+          ReportUnhandledException(e.obj());
+      }
+      // Notify the future.
+      job.future->storeResultUnlocked(result, ok);
     }
-    // Notify the future.
-    job.future->storeResultUnlocked(result, ok);
   }
 
   Kotlin_deinitRuntimeIfNeeded();
@@ -424,11 +419,9 @@ KInt currentWorker() {
 
 KInt schedule(KInt id, KInt transferMode, KRef producer, KNativePtr jobFunction) {
   Job job;
-  // Note that this is a bit hacky, as we must not auto-release jobArgumentRef,
+  ObjHolder holder;
-  // so we don't use ObjHolder.
+  WorkerLaunchpad(producer, holder.slot());
-  KRef jobArgumentRef = nullptr;
+  KNativePtr jobArgument = transfer(&holder, transferMode);
-  WorkerLaunchpad(producer, &jobArgumentRef);
-  KNativePtr jobArgument = transfer(jobArgumentRef, transferMode);
   Future* future = theState()->addJobToWorkerUnlocked(id, jobFunction, jobArgument, false, transferMode);
   if (future == nullptr) ThrowWorkerInvalidState();
   return future->id();
@@ -462,12 +455,13 @@ OBJ_GETTER(attachObjectGraphInternal, KNativePtr stable) {
 }
 
 KNativePtr detachObjectGraphInternal(KInt transferMode, KRef producer) {
-   KRef ref = nullptr;
+   ObjHolder result;
-   WorkerLaunchpad(producer, &ref);
+   WorkerLaunchpad(producer, result.slot());
-   if (ref != nullptr) {
+   if (result.obj() != nullptr) {
-     return transfer(ref, transferMode);
+     return transfer(&result, transferMode);
-   } else
+   } else {
      return nullptr;
+   }
 }
 
 #else

runtime/src/main/kotlin/kotlin/native/Annotations.kt

@@ -24,6 +24,14 @@ public annotation class SymbolName(val name: String)
 @Retention(AnnotationRetention.BINARY)
 public annotation class Retain
 
+/**
+ * Preserve the function entry point during global optimizations, only for the given target.
+ */
+@Target(AnnotationTarget.FUNCTION, AnnotationTarget.CLASS)
+@Retention(AnnotationRetention.BINARY)
+public annotation class RetainForTarget(val target: String)
+
+
 // TODO: merge with [kotlin.jvm.Throws]
 /**
  * This annotation indicates what exceptions should be declared by a function when compiled to a platform method.