Extract memory manager into a separate module (#4446)

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -202,6 +202,7 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                         MemoryModel.RELAXED
                     }
                     "strict" -> MemoryModel.STRICT
+                    "experimental" -> MemoryModel.EXPERIMENTAL
                     else -> {
                         configuration.report(ERROR, "Unsupported memory model ${arguments.memoryModel}")
                         MemoryModel.STRICT

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt

@@ -47,7 +47,7 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
     @Argument(value = "-manifest", valueDescription = "<path>", description = "Provide a maniferst addend file")
     var manifestFile: String? = null
 
-    @Argument(value="-memory-model", valueDescription = "<model>", description = "Memory model to use, 'strict' and 'relaxed' are currently supported")
+    @Argument(value="-memory-model", valueDescription = "<model>", description = "Memory model to use, 'strict', 'relaxed' and 'experimental' are currently supported")
     var memoryModel: String? = "strict"
 
     @Argument(value="-module-name", deprecatedName = "-module_name", valueDescription = "<name>", description = "Specify a name for the compilation module")

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfig.kt

@@ -115,17 +115,53 @@ class KonanConfig(val project: Project, val configuration: CompilerConfiguration
 
     internal val runtimeNativeLibraries: List<String> = mutableListOf<String>().apply {
         add(if (debug) "debug.bc" else "release.bc")
-        add(if (memoryModel == MemoryModel.STRICT) "strict.bc" else "relaxed.bc")
+        val effectiveMemoryModel = when (memoryModel) {
-        if (shouldCoverLibraries || shouldCoverSources) add("profileRuntime.bc")
+            MemoryModel.STRICT -> MemoryModel.STRICT
-        if (configuration.get(KonanConfigKeys.ALLOCATION_MODE) == "mimalloc") {
+            MemoryModel.RELAXED -> MemoryModel.RELAXED
-            if (!target.supportsMimallocAllocator()) {
+            MemoryModel.EXPERIMENTAL -> {
+                if (!target.supportsMimallocAllocator()) {
+                    configuration.report(CompilerMessageSeverity.STRONG_WARNING,
+                            "Experimental memory model requires mimalloc allocator. Used strict memory model.")
+                    MemoryModel.STRICT
+                } else if (!target.supportsThreads()) {
+                    configuration.report(CompilerMessageSeverity.STRONG_WARNING,
+                            "Experimental memory model requires threads, which are not supported on target ${target.name}. Used strict memory model.")
+                    MemoryModel.STRICT
+                } else {
+                    MemoryModel.EXPERIMENTAL
+                }
+            }
+        }
+        val useMimalloc = if (effectiveMemoryModel == MemoryModel.EXPERIMENTAL) {
+            true // we already checked that target supports mimalloc.
+        } else if (configuration.get(KonanConfigKeys.ALLOCATION_MODE) == "mimalloc") {
+            if (target.supportsMimallocAllocator()) {
+                true
+            } else {
                 configuration.report(CompilerMessageSeverity.STRONG_WARNING,
                         "Mimalloc allocator isn't supported on target ${target.name}. Used standard mode.")
-                add("std_alloc.bc")
+                false
-            } else {
-                add("opt_alloc.bc")
-                add("mimalloc.bc")
             }
+        } else {
+            false
+        }
+        when (effectiveMemoryModel) {
+            MemoryModel.STRICT -> {
+                add("strict.bc")
+                add("legacy_memory_manager.bc")
+            }
+            MemoryModel.RELAXED -> {
+                add("relaxed.bc")
+                add("legacy_memory_manager.bc")
+            }
+            MemoryModel.EXPERIMENTAL -> {
+                add("experimental_memory_manager.bc")
+            }
+        }
+        if (shouldCoverLibraries || shouldCoverSources) add("profileRuntime.bc")
+        if (useMimalloc) {
+            add("opt_alloc.bc")
+            add("mimalloc.bc")
         } else {
             add("std_alloc.bc")
         }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/MemoryModel.kt

@@ -4,7 +4,8 @@
  */
 package org.jetbrains.kotlin.backend.konan
 
-enum class MemoryModel(val suffix: String) {
+enum class MemoryModel {
-    STRICT("Strict"),
+    STRICT,
-    RELAXED("Relaxed")
+    RELAXED,
+    EXPERIMENTAL,
 }

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -494,23 +494,20 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
     }
 
     private fun importRtFunction(name: String) = importFunction(name, runtime.llvmModule)
-    private fun importModelSpecificRtFunction(name: String) =
-            importRtFunction(name + context.memoryModel.suffix)
 
     private fun importRtGlobal(name: String) = importGlobal(name, runtime.llvmModule)
 
-    val allocInstanceFunction = importModelSpecificRtFunction("AllocInstance")
+    val allocInstanceFunction = importRtFunction("AllocInstance")
-    val allocArrayFunction = importModelSpecificRtFunction("AllocArrayInstance")
+    val allocArrayFunction = importRtFunction("AllocArrayInstance")
-    val initInstanceFunction = importModelSpecificRtFunction("InitInstance")
+    val initInstanceFunction = importRtFunction("InitInstance")
-    val initSharedInstanceFunction = importModelSpecificRtFunction("InitSharedInstance")
+    val initSharedInstanceFunction = importRtFunction("InitSharedInstance")
-    val updateHeapRefFunction = importModelSpecificRtFunction("UpdateHeapRef")
+    val updateHeapRefFunction = importRtFunction("UpdateHeapRef")
-    val releaseHeapRefFunction = importModelSpecificRtFunction("ReleaseHeapRef")
+    val updateStackRefFunction = importRtFunction("UpdateStackRef")
-    val updateStackRefFunction = importModelSpecificRtFunction("UpdateStackRef")
+    val updateReturnRefFunction = importRtFunction("UpdateReturnRef")
-    val updateReturnRefFunction = importModelSpecificRtFunction("UpdateReturnRef")
     val zeroHeapRefFunction = importRtFunction("ZeroHeapRef")
     val zeroArrayRefsFunction = importRtFunction("ZeroArrayRefs")
-    val enterFrameFunction = importModelSpecificRtFunction("EnterFrame")
+    val enterFrameFunction = importRtFunction("EnterFrame")
-    val leaveFrameFunction = importModelSpecificRtFunction("LeaveFrame")
+    val leaveFrameFunction = importRtFunction("LeaveFrame")
     val lookupOpenMethodFunction = importRtFunction("LookupOpenMethod")
     val lookupInterfaceTableRecord = importRtFunction("LookupInterfaceTableRecord")
     val isInstanceFunction = importRtFunction("IsInstance")

runtime/build.gradle.kts

@@ -37,7 +37,9 @@ bitcode {
             "${target}Relaxed",
             "${target}ProfileRuntime",
             "${target}Objc",
-            "${target}ExceptionsSupport"
+            "${target}ExceptionsSupport",
+            "${target}LegacyMemoryManager",
+            "${target}ExperimentalMemoryManager"
         )
         includeRuntime()
         linkerArgs.add(project.file("../common/build/bitcode/main/$target/hash.bc").path)
@@ -94,6 +96,14 @@ bitcode {
         dependsOn("downloadGoogleTest")
         headersDirs += googletest.headersDirs
     }
+
+    create("legacy_memory_manager", file("src/legacymm")) {
+        includeRuntime()
+    }
+
+    create("experimental_memory_manager", file("src/mm")) {
+        includeRuntime()
+    }
 }
 
 targetList.forEach { targetName ->
@@ -103,6 +113,7 @@ targetList.forEach { targetName ->
             "${targetName}StdAllocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
+                "${targetName}LegacyMemoryManager",
                 "${targetName}Strict",
                 "${targetName}Release",
                 "${targetName}StdAlloc"
@@ -117,6 +128,7 @@ targetList.forEach { targetName ->
             "${targetName}MimallocRuntimeTests",
             listOf(
                 "${targetName}Runtime",
+                "${targetName}LegacyMemoryManager",
                 "${targetName}Strict",
                 "${targetName}Release",
                 "${targetName}Mimalloc",
@@ -126,9 +138,23 @@ targetList.forEach { targetName ->
         includeRuntime()
     }
 
+    createTestTask(
+            project,
+            "ExperimentalMM",
+            "${targetName}ExperimentalMMRuntimeTests",
+            listOf(
+                "${targetName}Runtime",
+                "${targetName}ExperimentalMemoryManager",
+                "${targetName}Release",
+                "${targetName}Mimalloc",
+                "${targetName}OptAlloc"
+            )
+    )
+
     tasks.register("${targetName}RuntimeTests") {
         dependsOn("${targetName}StdAllocRuntimeTests")
         dependsOn("${targetName}MimallocRuntimeTests")
+        dependsOn("${targetName}ExperimentalMMRuntimeTests")
     }
 }
 
@@ -148,6 +174,10 @@ val hostMimallocRuntimeTests by tasks.registering {
     dependsOn("${hostName}MimallocRuntimeTests")
 }
 
+val hostExperimentalMMRuntimeTests by tasks.registering {
+    dependsOn("${hostName}ExperimentalMMRuntimeTests")
+}
+
 val assemble by tasks.registering {
     dependsOn(tasks.withType(CompileToBitcode::class).matching {
         it.outputGroup == "main"

runtime/src/legacymm/cpp/CyclicCollector.cpp

@@ -190,7 +190,7 @@ class CyclicCollector {
          sideRefCounts.clear();
          for (auto* root: rootset_) {
            // We only care about frozen values here, as only they could become part of shared cycles.
-           if (!root->container()->frozen()) continue;
+           if (!containerFor(root)->frozen()) continue;
            COLLECTOR_LOG("process root %p\n", root);
            toVisit.push_back(root);
            sideRefCounts[root] = 0;
@@ -204,7 +204,7 @@ class CyclicCollector {
            auto* obj = toVisit.front();
            toVisit.pop_front();
            COLLECTOR_LOG("visit %s%p\n", isAtomicReference(obj) ? "atomic " : "", obj);
-           auto* objContainer = obj->container();
+           auto* objContainer = containerFor(obj);
            if (objContainer == nullptr) continue;  // Permanent object.
            RuntimeCheck(objContainer->shareable(), "Must be shareable");
            if (visited.count(obj) == 0) {
@@ -216,7 +216,7 @@ class CyclicCollector {
                   int increment;
                   // We shall not account for edges inside the same frozen container, unless it originates
                   // from an atomic reference.
-                  if (isAtomicReference(obj) || (obj->container() != ref->container())) {
+                  if (isAtomicReference(obj) || (containerFor(obj) != containerFor(ref))) {
                     COLLECTOR_LOG("counting %p -> %p\n", obj, ref)
                     increment = 1;
                   } else {
@@ -234,7 +234,7 @@ class CyclicCollector {
          toVisit.clear();
          for (auto it: sideRefCounts) {
            auto* obj = it.first;
-           auto* objContainer = obj->container();
+           auto* objContainer = containerFor(obj);
            if (objContainer == nullptr) continue;  // Permanent object.
            int refCount;
            // If object is in aggregated container - sum up RC for all elements.
@@ -260,7 +260,7 @@ class CyclicCollector {
          while (toVisit.size() > 0)  {
            auto* obj = toVisit.front();
            toVisit.pop_front();
-           auto* objContainer = obj->container();
+           auto* objContainer = containerFor(obj);
            if (objContainer == nullptr) continue;  // Permanent object.
            RuntimeCheck(objContainer->shareable(), "Must be shareable");
            sideRefCounts[obj] = -1;
@@ -290,7 +290,7 @@ class CyclicCollector {
              restartCount++;
              goto restart;
            }
-           auto* objContainer = obj->container();
+           auto* objContainer = containerFor(obj);
            if (!objContainer->frozen()) continue;
            RuntimeAssert(objContainer->objectCount() == 1, "Must be single object");
            COLLECTOR_LOG("for %p inner %d actual %d\n", obj, it.second, objContainer->refCount());

runtime/src/legacymm/cpp/Memory.cpp

@@ -46,6 +46,11 @@
 #include "Runtime.h"
 #include "Utils.h"
 #include "WorkerBoundReference.h"
+#include "Weak.h"
+
+#ifdef KONAN_OBJC_INTEROP
+#include "ObjCMMAPI.h"
+#endif
 
 // If garbage collection algorithm for cyclic garbage to be used.
 // We are using the Bacon's algorithm for GC, see
@@ -66,6 +71,8 @@
 
 namespace {
 
+typedef uint32_t container_size_t;
+
 // Granularity of arena container chunks.
 constexpr container_size_t kContainerAlignment = 1024;
 // Single object alignment.
@@ -322,7 +329,7 @@ public:
 
   static int toIndex(const ObjHeader* obj, int stack) {
     if (reinterpret_cast<uintptr_t>(obj) > 1)
-        return toIndex(obj->container(), stack);
+        return toIndex(containerFor(obj), stack);
     else
         return 4 + stack * 6;
   }
@@ -430,8 +437,58 @@ inline bool isShareable(ContainerHeader* container) {
     return container == nullptr || container->shareable();
 }
 
+void setContainerFor(ObjHeader* obj, ContainerHeader* container) {
+  obj->meta_object()->container_ = container;
+  obj->typeInfoOrMeta_ = setPointerBits(obj->typeInfoOrMeta_, OBJECT_TAG_NONTRIVIAL_CONTAINER);
+}
+
 }  // namespace
 
+ContainerHeader* containerFor(const ObjHeader* obj) {
+  unsigned bits = getPointerBits(obj->typeInfoOrMeta_, OBJECT_TAG_MASK);
+  if ((bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) == 0)
+    return reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(obj)) - 1;
+  if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0)
+    return nullptr;
+  return (reinterpret_cast<MetaObjHeader*>(clearPointerBits(obj->typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_;
+}
+
+ALWAYS_INLINE bool isFrozen(const ObjHeader* obj) {
+    return containerFor(obj)->frozen();
+}
+
+ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {
+    auto* container = containerFor(obj);
+    return container == nullptr || container->frozen();
+}
+
+ALWAYS_INLINE bool isShareable(const ObjHeader* obj) {
+    return containerFor(obj)->shareable();
+}
+
+ObjHeader** ObjHeader::GetWeakCounterLocation() {
+    return &this->meta_object()->WeakReference.counter_;
+}
+
+#if KONAN_OBJC_INTEROP
+
+void* ObjHeader::GetAssociatedObject() {
+    if (!has_meta_object()) {
+      return nullptr;
+    }
+    return this->meta_object()->associatedObject_;
+}
+
+void** ObjHeader::GetAssociatedObjectLocation() {
+    return &this->meta_object()->associatedObject_;
+}
+
+void ObjHeader::SetAssociatedObject(void* obj) {
+    this->meta_object()->associatedObject_ = obj;
+}
+
+#endif // KONAN_OBJC_INTEROP
+
 class ForeignRefManager {
  public:
   static ForeignRefManager* create() {
@@ -590,7 +647,7 @@ struct MemoryState {
     state->statistic.incFree(container);
   #define OBJECT_ALLOC_STAT(state, size, object) \
     state->statistic.incAlloc(size, object); \
-    state->statistic.incAddRef(object->container(), 0, 0);
+    state->statistic.incAddRef(containerFor(object), 0, 0);
   #define UPDATE_REF_STAT(state, oldRef, newRef, slot, stack) \
     state->statistic.incUpdateRef(oldRef, newRef, stack);
   #define UPDATE_ADDREF_STAT(state, obj, atomic, stack) \
@@ -769,7 +826,7 @@ class ArenaContainer {
 
   void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) {
     obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
-    obj->setContainer(currentChunk_->asHeader());
+    setContainerFor(obj, currentChunk_->asHeader());
     // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC.
   }
 
@@ -813,7 +870,7 @@ inline container_size_t alignUp(container_size_t size, int alignment) {
 
 inline ContainerHeader* realShareableContainer(ContainerHeader* container) {
   RuntimeAssert(container->shareable(), "Only makes sense on shareable objects");
-  return reinterpret_cast<ObjHeader*>(container + 1)->container();
+  return containerFor(reinterpret_cast<ObjHeader*>(container + 1));
 }
 
 inline uint32_t arrayObjectSize(const TypeInfo* typeInfo, uint32_t count) {
@@ -894,7 +951,7 @@ inline FrameOverlay* asFrameOverlay(ObjHeader** slot) {
 }
 
 inline bool isRefCounted(KConstRef object) {
-  return isFreeable(object->container());
+  return isFreeable(containerFor(object));
 }
 
 inline void lock(KInt* spinlock) {
@@ -973,7 +1030,7 @@ ContainerHeader* allocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& c
     *place++ = container;
     // Set link to the new container.
     auto* obj = reinterpret_cast<ObjHeader*>(container + 1);
-    obj->setContainer(superContainer);
+    setContainerFor(obj, superContainer);
     MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer);
   }
   superContainer->setObjectCount(componentSize);
@@ -1012,7 +1069,7 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) {
       return true;
     }
     traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) {
-        auto* child = ref->container();
+        auto* child = containerFor(ref);
         if (!isShareable(child) && (visited->count(child) == 0)) {
            toVisit.push_front(child);
         }
@@ -1155,7 +1212,7 @@ void depthFirstTraversal(ContainerHeader* start, bool* hasCycles,
           *firstBlocker = obj;
           return;
       }
-      ContainerHeader* objContainer = obj->container();
+      ContainerHeader* objContainer = containerFor(obj);
       if (canFreeze(objContainer)) {
         // Marked GREY, there's cycle.
         if (objContainer->seen()) *hasCycles = true;
@@ -1384,7 +1441,7 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet*
   seen->insert(header);
   if (!isAggregatingFrozenContainer(header)) {
     traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) {
-      auto* child = ref->container();
+      auto* child = containerFor(ref);
       RuntimeAssert(!isArena(child), "A reference to local object is encountered");
       if (child != nullptr && (seen->count(child) == 0)) {
         dumpWorker(prefix, child, seen);
@@ -1433,7 +1490,7 @@ void markGray(ContainerHeader* start) {
     }
 
     traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
-      auto* childContainer = ref->container();
+      auto* childContainer = containerFor(ref);
       RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
       if (!isShareable(childContainer)) {
         childContainer->decRefCount<false>();
@@ -1460,7 +1517,7 @@ void scanBlack(ContainerHeader* start) {
       container->unMark();
     }
     traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
-        auto childContainer = ref->container();
+        auto childContainer = containerFor(ref);
         RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
         if (!isShareable(childContainer)) {
           childContainer->incRefCount<false>();
@@ -1539,7 +1596,7 @@ void scan(ContainerHeader* start) {
      }
      container->setColorAssertIfGreen(CONTAINER_TAG_GC_WHITE);
      traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
-       auto* childContainer = ref->container();
+       auto* childContainer = containerFor(ref);
        RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
        if (!isShareable(childContainer)) {
          toVisit.push_front(childContainer);
@@ -1560,7 +1617,7 @@ void collectWhite(MemoryState* state, ContainerHeader* start) {
      traverseContainerObjectFields(container, [&toVisit](ObjHeader** location) {
         auto* ref = *location;
         if (ref == nullptr) return;
-        auto* childContainer = ref->container();
+        auto* childContainer = containerFor(ref);
         RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
         if (isShareable(childContainer)) {
           ZeroHeapRef(location);
@@ -1603,7 +1660,7 @@ inline void addHeapRef(ContainerHeader* container) {
 }
 
 inline void addHeapRef(const ObjHeader* header) {
-  auto* container = header->container();
+  auto* container = containerFor(header);
   if (container != nullptr)
     addHeapRef(const_cast<ContainerHeader*>(container));
 }
@@ -1627,7 +1684,7 @@ inline bool tryAddHeapRef(ContainerHeader* container) {
 }
 
 inline bool tryAddHeapRef(const ObjHeader* header) {
-  auto* container = header->container();
+  auto* container = containerFor(header);
   return (container != nullptr) ? tryAddHeapRef(container) : true;
 }
 
@@ -1645,7 +1702,7 @@ inline void releaseHeapRef(ContainerHeader* container) {
 
 template <bool Strict, bool CanCollect = true>
 inline void releaseHeapRef(const ObjHeader* header) {
-  auto* container = header->container();
+  auto* container = containerFor(header);
   if (container != nullptr)
     releaseHeapRef<Strict, CanCollect>(const_cast<ContainerHeader*>(container));
 }
@@ -1685,7 +1742,7 @@ void incrementStack(MemoryState* state) {
     while (current < end) {
       ObjHeader* obj = *current++;
       if (obj != nullptr) {
-        auto* container = obj->container();
+        auto* container = containerFor(obj);
         if (container == nullptr) continue;
         if (container->shareable()) {
           incrementRC<true>(container);
@@ -1713,7 +1770,7 @@ void processDecrements(MemoryState* state) {
   }
 
   state->foreignRefManager->processEnqueuedReleaseRefsWith([](ObjHeader* obj) {
-    ContainerHeader* container = obj->container();
+    ContainerHeader* container = containerFor(obj);
     if (container != nullptr) decrementRC(container);
   });
   state->gcSuspendCount--;
@@ -1730,7 +1787,7 @@ void decrementStack(MemoryState* state) {
       ObjHeader* obj = *current++;
       if (obj != nullptr) {
         MEMORY_LOG("decrement stack %p\n", obj)
-        auto* container = obj->container();
+        auto* container = containerFor(obj);
         if (container != nullptr)
           enqueueDecrementRC</* CanCollect = */ false>(container);
       }
@@ -1897,7 +1954,7 @@ bool isForeignRefAccessible(ObjHeader* object, ForeignRefManager* manager) {
   }
 
   // Note: getting container and checking it with 'isShareable()' is supposed to be correct even for unowned object.
-  return isShareable(object->container());
+  return isShareable(containerFor(object));
 }
 
 void deinitForeignRef(ObjHeader* object, ForeignRefManager* manager) {
@@ -2322,7 +2379,7 @@ OBJ_GETTER(swapHeapRefLocked,
 
   if (IsStrictMemoryModel && shallRemember && oldValue != nullptr && oldValue != expectedValue) {
     // Only remember container if it is not known to this thread (i.e. != expectedValue).
-    rememberNewContainer(oldValue->container());
+    rememberNewContainer(containerFor(oldValue));
   }
   unlock(spinlock);
 
@@ -2357,7 +2414,7 @@ OBJ_GETTER(readHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t*
   UpdateReturnRef(OBJ_RESULT, value);
 #if USE_GC
   if (IsStrictMemoryModel && shallRemember && value != nullptr) {
-    auto* container = value->container();
+    auto* container = containerFor(value);
     rememberNewContainer(container);
   }
 #endif  // USE_GC
@@ -2373,7 +2430,7 @@ OBJ_GETTER(readHeapRefNoLock, ObjHeader* object, KInt index) {
 #if USE_GC
   if (IsStrictMemoryModel && (value != nullptr)) {
     // Maybe not so good to do that under lock.
-    rememberNewContainer(value->container());
+    rememberNewContainer(containerFor(value));
   }
 #endif  // USE_GC
   RETURN_OBJ(value);
@@ -2506,7 +2563,7 @@ KBoolean getTuneGCThreshold() {
 
 KNativePtr createStablePointer(KRef any) {
   if (any == nullptr) return nullptr;
-  MEMORY_LOG("CreateStablePointer for %p rc=%d\n", any, any->container() ? any->container()->refCount() : 0)
+  MEMORY_LOG("CreateStablePointer for %p rc=%d\n", any, containerFor(any) ? containerFor(any)->refCount() : 0)
   addHeapRef(any);
   return reinterpret_cast<KNativePtr>(any);
 }
@@ -2527,7 +2584,7 @@ OBJ_GETTER(adoptStablePointer, KNativePtr pointer) {
   synchronize();
   KRef ref = reinterpret_cast<KRef>(pointer);
   MEMORY_LOG("adopting stable pointer %p, rc=%d\n", \
-     ref, (ref && ref->container()) ? ref->container()->refCount() : -1)
+     ref, (ref && containerFor(ref)) ? containerFor(ref)->refCount() : -1)
   UpdateReturnRef(OBJ_RESULT, ref);
   DisposeStablePointer(pointer);
   return ref;
@@ -2538,7 +2595,7 @@ bool clearSubgraphReferences(ObjHeader* root, bool checked) {
   MEMORY_LOG("ClearSubgraphReferences %p\n", root)
   if (root == nullptr) return true;
   auto state = memoryState;
-  auto* container = root->container();
+  auto* container = containerFor(root);
 
   if (isShareable(container))
     // We assume, that frozen/shareable objects can be safely passed and not present
@@ -2624,7 +2681,7 @@ void freezeAcyclic(ContainerHeader* rootContainer, ContainerHeaderSet* newlyFroz
     MEMORY_LOG("freezing %p\n", current)
     current->freeze();
     traverseContainerReferredObjects(current, [&queue](ObjHeader* obj) {
-        ContainerHeader* objContainer = obj->container();
+        ContainerHeader* objContainer = containerFor(obj);
         if (canFreeze(objContainer)) {
           if (objContainer->marked())
             queue.push_back(objContainer);
@@ -2642,11 +2699,11 @@ void freezeCyclic(ObjHeader* root,
   while (!queue.empty()) {
     ObjHeader* current = queue.front();
     queue.pop_front();
-    ContainerHeader* currentContainer = current->container();
+    ContainerHeader* currentContainer = containerFor(current);
     currentContainer->unMark();
     reversedEdges.emplace(currentContainer, KStdVector<ContainerHeader*>(0));
     traverseContainerReferredObjects(currentContainer, [current, currentContainer, &queue, &reversedEdges](ObjHeader* obj) {
-          ContainerHeader* objContainer = obj->container();
+          ContainerHeader* objContainer = containerFor(obj);
           if (canFreeze(objContainer)) {
             if (objContainer->marked())
               queue.push_back(obj);
@@ -2687,7 +2744,7 @@ void freezeCyclic(ObjHeader* root,
         continue;
       }
       traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) {
-          auto* container = obj->container();
+          auto* container = containerFor(obj);
           if (canFreeze(container))
             ++internalRefsCount;
         });
@@ -2739,7 +2796,7 @@ void runFreezeHooksRecursive(ObjHeader* root) {
     traverseReferredObjects(obj, [&seen, &toVisit](ObjHeader* field) {
       auto wasNotSeenYet = seen.insert(field).second;
       // Only iterating on unseen objects which containers will get frozen by freezeCyclic or freezeAcyclic.
-      if (wasNotSeenYet && canFreeze(field->container())) {
+      if (wasNotSeenYet && canFreeze(containerFor(field))) {
         toVisit.push_back(field);
       }
     });
@@ -2773,7 +2830,7 @@ void freezeSubgraph(ObjHeader* root) {
   if (root == nullptr) return;
   // First check that passed object graph has no cycles.
   // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir.
-  ContainerHeader* rootContainer = root->container();
+  ContainerHeader* rootContainer = containerFor(root);
   if (isPermanentOrFrozen(rootContainer)) return;
 
   MEMORY_LOG("Run freeze hooks on subgraph of %p\n", root);
@@ -2823,7 +2880,7 @@ void freezeSubgraph(ObjHeader* root) {
 }
 
 void ensureNeverFrozen(ObjHeader* object) {
-   auto* container = object->container();
+   auto* container = containerFor(object);
    if (container == nullptr || container->frozen())
       ThrowFreezingException(object, object);
    // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency,
@@ -2832,7 +2889,7 @@ void ensureNeverFrozen(ObjHeader* object) {
 }
 
 void shareAny(ObjHeader* obj) {
-  auto* container = obj->container();
+  auto* container = containerFor(obj);
   if (isShareable(container)) return;
   RuntimeCheck(container->objectCount() == 1, "Must be a single object container");
   container->makeShared();
@@ -3165,8 +3222,8 @@ bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) {
 
 void AdoptReferenceFromSharedVariable(ObjHeader* object) {
 #if USE_GC
-  if (IsStrictMemoryModel && object != nullptr && isShareable(object->container()))
+  if (IsStrictMemoryModel && object != nullptr && isShareable(containerFor(object)))
-    rememberNewContainer(object->container());
+    rememberNewContainer(containerFor(object));
 #endif  // USE_GC
 }
 
@@ -3444,7 +3501,7 @@ void FreezeSubgraph(ObjHeader* root) {
 // If object is frozen or permanent, an exception is thrown.
 void MutationCheck(ObjHeader* obj) {
   if (obj->local()) return;
-  auto* container = obj->container();
+  auto* container = containerFor(obj);
   if (container == nullptr || container->frozen())
     ThrowInvalidMutabilityException(obj);
 }
@@ -3544,7 +3601,7 @@ void Kotlin_native_internal_GC_setCyclicCollector(KRef gc, KBoolean value) {
 }
 
 bool Kotlin_Any_isShareable(KRef thiz) {
-    return thiz == nullptr || isShareable(thiz->container());
+    return thiz == nullptr || isShareable(containerFor(thiz));
 }
 
 void PerformFullGC() {

runtime/src/legacymm/cpp/MemoryPrivate.hpp

@@ -0,0 +1,337 @@
+/*
+ * Copyright 2010-2017 JetBrains s.r.o.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RUNTIME_MEMORYPRIVATE_HPP
+#define RUNTIME_MEMORYPRIVATE_HPP
+
+#include "Memory.h"
+
+typedef enum {
+  // Those bit masks are applied to refCount_ field.
+  // Container is normal thread-local container.
+  CONTAINER_TAG_LOCAL = 0,
+  // Container is frozen, could only refer to other frozen objects.
+  // Refcounter update is atomics.
+  CONTAINER_TAG_FROZEN = 1 | 1,  // shareable
+  // Stack container, no need to free, children cleanup still shall be there.
+  CONTAINER_TAG_STACK = 2,
+  // Atomic container, reference counter is atomically updated.
+  CONTAINER_TAG_SHARED = 3 | 1,  // shareable
+  // Shift to get actual counter.
+  CONTAINER_TAG_SHIFT = 2,
+  // Actual value to increment/decrement container by. Tag is in lower bits.
+  CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
+  // Mask for container type.
+  CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
+
+  // Shift to get actual object count, if has it.
+  CONTAINER_TAG_GC_SHIFT     = 7,
+  CONTAINER_TAG_GC_MASK      = (1 << CONTAINER_TAG_GC_SHIFT) - 1,
+  CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT,
+  // Color mask of a container.
+  CONTAINER_TAG_COLOR_SHIFT   = 3,
+  CONTAINER_TAG_GC_COLOR_MASK = (1 << CONTAINER_TAG_COLOR_SHIFT) - 1,
+  // Colors.
+  // In use or free.
+  CONTAINER_TAG_GC_BLACK  = 0,
+  // Possible member of garbage cycle.
+  CONTAINER_TAG_GC_GRAY   = 1,
+  // Member of garbage cycle.
+  CONTAINER_TAG_GC_WHITE  = 2,
+  // Possible root of cycle.
+  CONTAINER_TAG_GC_PURPLE = 3,
+  // Acyclic.
+  CONTAINER_TAG_GC_GREEN  = 4,
+  // Orange and red are currently unused.
+  // Candidate cycle awaiting epoch.
+  CONTAINER_TAG_GC_ORANGE = 5,
+  // Candidate cycle awaiting sigma computation.
+  CONTAINER_TAG_GC_RED    = 6,
+  // Individual state bits used during GC and freezing.
+  CONTAINER_TAG_GC_MARKED   = 1 << CONTAINER_TAG_COLOR_SHIFT,
+  CONTAINER_TAG_GC_BUFFERED = 1 << (CONTAINER_TAG_COLOR_SHIFT + 1),
+  CONTAINER_TAG_GC_SEEN     = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2),
+  // If indeed has more that one object.
+  CONTAINER_TAG_GC_HAS_OBJECT_COUNT = 1 << (CONTAINER_TAG_COLOR_SHIFT + 3)
+} ContainerTag;
+
+// Header of all container objects. Contains reference counter.
+struct ContainerHeader {
+  // Reference counter of container. Uses CONTAINER_TAG_SHIFT, lower bits of counter
+  // for container type (for polymorphism in ::Release()).
+  uint32_t refCount_;
+  // Number of objects in the container.
+  uint32_t objectCount_;
+
+  inline bool local() const {
+      return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_LOCAL;
+  }
+
+  inline bool frozen() const {
+    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN;
+  }
+
+  inline void freeze() {
+    refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_FROZEN;
+  }
+
+  inline void makeShared() {
+      refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_SHARED;
+  }
+
+  inline bool shared() const {
+    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_SHARED;
+  }
+
+  inline bool shareable() const {
+      return (tag() & 1) != 0; // CONTAINER_TAG_FROZEN || CONTAINER_TAG_SHARED
+  }
+
+  inline bool stack() const {
+    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK;
+  }
+
+  inline int refCount() const {
+    return (int)refCount_ >> CONTAINER_TAG_SHIFT;
+  }
+
+  inline void setRefCount(unsigned refCount) {
+    refCount_ = tag() | (refCount << CONTAINER_TAG_SHIFT);
+  }
+
+  template <bool Atomic>
+  inline void incRefCount() {
+#ifdef KONAN_NO_THREADS
+    refCount_ += CONTAINER_TAG_INCREMENT;
+#else
+    if (Atomic)
+      __sync_add_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT);
+    else
+      refCount_ += CONTAINER_TAG_INCREMENT;
+#endif
+  }
+
+  template <bool Atomic>
+  inline bool tryIncRefCount() {
+    if (Atomic) {
+      while (true) {
+        uint32_t currentRefCount_ = refCount_;
+        if (((int)currentRefCount_ >> CONTAINER_TAG_SHIFT) > 0) {
+          if (compareAndSet(&refCount_, currentRefCount_, currentRefCount_ + CONTAINER_TAG_INCREMENT)) {
+            return true;
+          }
+        } else {
+          return false;
+        }
+      }
+    } else {
+      // Note: tricky case here is doing this during cycle collection.
+      // This can actually happen due to deallocation hooks.
+      // Fortunately by this point reference counts have been made precise again.
+      if (refCount() > 0) {
+        incRefCount</* Atomic = */ false>();
+        return true;
+      } else {
+        return false;
+      }
+    }
+  }
+
+  template <bool Atomic>
+  inline int decRefCount() {
+#ifdef KONAN_NO_THREADS
+    int value = refCount_ -= CONTAINER_TAG_INCREMENT;
+#else
+    int value = Atomic ?
+       __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT;
+#endif
+    return value >> CONTAINER_TAG_SHIFT;
+  }
+
+  inline int decRefCount() {
+  #ifdef KONAN_NO_THREADS
+      int value = refCount_ -= CONTAINER_TAG_INCREMENT;
+  #else
+      int value = shareable() ?
+         __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT;
+  #endif
+      return value >> CONTAINER_TAG_SHIFT;
+  }
+
+  inline unsigned tag() const {
+    return refCount_ & CONTAINER_TAG_MASK;
+  }
+
+  inline unsigned objectCount() const {
+    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0 ?
+        (objectCount_ >> CONTAINER_TAG_GC_SHIFT) : 1;
+  }
+
+  inline void incObjectCount() {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0, "Must have object count");
+    objectCount_ += CONTAINER_TAG_GC_INCREMENT;
+  }
+
+  inline void setObjectCount(int count) {
+    if (count == 1) {
+      objectCount_ &= ~CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
+    } else {
+      objectCount_ = (count << CONTAINER_TAG_GC_SHIFT) | CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
+    }
+  }
+
+  inline unsigned containerSize() const {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must be single-object");
+    return (objectCount_ >> CONTAINER_TAG_GC_SHIFT);
+  }
+
+  inline void setContainerSize(unsigned size) {
+    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must not have object count");
+    objectCount_ = (objectCount_ & CONTAINER_TAG_GC_MASK) | (size << CONTAINER_TAG_GC_SHIFT);
+  }
+
+  inline bool hasContainerSize() {
+    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0;
+  }
+
+  inline unsigned color() const {
+    return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK;
+  }
+
+  inline void setColorAssertIfGreen(unsigned color) {
+    RuntimeAssert(this->color() != CONTAINER_TAG_GC_GREEN, "Must not be green");
+    setColorEvenIfGreen(color);
+  }
+
+  inline void setColorEvenIfGreen(unsigned color) {
+    // TODO: do we need atomic color update?
+    objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
+  }
+
+  inline void setColorUnlessGreen(unsigned color) {
+    // TODO: do we need atomic color update?
+    unsigned objectCount = objectCount_;
+    if ((objectCount & CONTAINER_TAG_GC_COLOR_MASK) != CONTAINER_TAG_GC_GREEN)
+        objectCount_ = (objectCount & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
+  }
+
+  inline bool buffered() const {
+    return (objectCount_ & CONTAINER_TAG_GC_BUFFERED) != 0;
+  }
+
+  inline void setBuffered() {
+    objectCount_ |= CONTAINER_TAG_GC_BUFFERED;
+  }
+
+  inline void resetBuffered() {
+    objectCount_ &= ~CONTAINER_TAG_GC_BUFFERED;
+  }
+
+  inline bool marked() const {
+    return (objectCount_ & CONTAINER_TAG_GC_MARKED) != 0;
+  }
+
+  inline void mark() {
+    objectCount_ |= CONTAINER_TAG_GC_MARKED;
+  }
+
+  inline void unMark() {
+    objectCount_ &= ~CONTAINER_TAG_GC_MARKED;
+  }
+
+  inline bool seen() const {
+    return (objectCount_ & CONTAINER_TAG_GC_SEEN) != 0;
+  }
+
+  inline void setSeen() {
+    objectCount_ |= CONTAINER_TAG_GC_SEEN;
+  }
+
+  inline void resetSeen() {
+    objectCount_ &= ~CONTAINER_TAG_GC_SEEN;
+  }
+
+  // Following operations only work on freed container which is in finalization queue.
+  // We cannot use 'this' here, as it conflicts with aliasing analysis in clang.
+  inline void setNextLink(ContainerHeader* next) {
+    *reinterpret_cast<ContainerHeader**>(this + 1) = next;
+  }
+
+  inline ContainerHeader* nextLink() {
+    return *reinterpret_cast<ContainerHeader**>(this + 1);
+  }
+};
+
+ALWAYS_INLINE ContainerHeader* containerFor(const ObjHeader* obj);
+
+// Header for the meta-object.
+struct MetaObjHeader {
+  // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
+  const TypeInfo* typeInfo_;
+  // Container pointer.
+  ContainerHeader* container_;
+
+#ifdef KONAN_OBJC_INTEROP
+  void* associatedObject_;
+#endif
+
+  // Flags for the object state.
+  int32_t flags_;
+
+  struct {
+    // Strong reference to the counter object.
+    ObjHeader* counter_;
+  } WeakReference;
+};
+
+extern "C" {
+
+#define MODEL_VARIANTS(returnType, name, ...)            \
+   returnType name##Strict(__VA_ARGS__) RUNTIME_NOTHROW; \
+   returnType name##Relaxed(__VA_ARGS__) RUNTIME_NOTHROW;
+
+OBJ_GETTER(AllocInstanceStrict, const TypeInfo* type_info) RUNTIME_NOTHROW;
+OBJ_GETTER(AllocInstanceRelaxed, const TypeInfo* type_info) RUNTIME_NOTHROW;
+
+OBJ_GETTER(AllocArrayInstanceStrict, const TypeInfo* type_info, int32_t elements);
+OBJ_GETTER(AllocArrayInstanceRelaxed, const TypeInfo* type_info, int32_t elements);
+
+OBJ_GETTER(InitInstanceStrict,
+    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+OBJ_GETTER(InitInstanceRelaxed,
+    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+
+OBJ_GETTER(InitSharedInstanceStrict,
+    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+OBJ_GETTER(InitSharedInstanceRelaxed,
+    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
+
+MODEL_VARIANTS(void, SetStackRef, ObjHeader** location, const ObjHeader* object);
+MODEL_VARIANTS(void, SetHeapRef, ObjHeader** location, const ObjHeader* object);
+MODEL_VARIANTS(void, ZeroStackRef, ObjHeader** location);
+MODEL_VARIANTS(void, UpdateStackRef, ObjHeader** location, const ObjHeader* object);
+MODEL_VARIANTS(void, UpdateHeapRef, ObjHeader** location, const ObjHeader* object);
+MODEL_VARIANTS(void, UpdateHeapRefIfNull, ObjHeader** location, const ObjHeader* object);
+MODEL_VARIANTS(void, UpdateReturnRef, ObjHeader** returnSlot, const ObjHeader* object);
+MODEL_VARIANTS(void, EnterFrame, ObjHeader** start, int parameters, int count);
+MODEL_VARIANTS(void, LeaveFrame, ObjHeader** start, int parameters, int count);
+
+MODEL_VARIANTS(void, ReleaseHeapRef, const ObjHeader* object);
+MODEL_VARIANTS(void, ReleaseHeapRefNoCollect, const ObjHeader* object);
+
+}  // extern "C"
+
+#endif // RUNTIME_MEMORYPRIVATE_HPP

runtime/src/main/cpp/Arrays.cpp

@@ -29,8 +29,8 @@ namespace {
 
 ALWAYS_INLINE inline void mutabilityCheck(KConstRef thiz) {
   // TODO: optimize it!
-  if (!thiz->local() && thiz->container()->frozen()) {
+  if (!thiz->local() && isFrozen(thiz)) {
-    ThrowInvalidMutabilityException(thiz);
+      ThrowInvalidMutabilityException(thiz);
   }
 }
 

runtime/src/main/cpp/CompilerExport.cpp

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "Memory.h"
+
+namespace {
+
+template <typename T>
+T defaultValue() {
+    return T();
+}
+
+template <typename Ret, typename... Args>
+void ensureUsed(Ret (*f)(Args...)) {
+    f(defaultValue<Args>()...);
+}
+
+} // namespace
+
+// This is a hack to force clang to emit possibly unused declarations.
+// TODO: Make sure this function gets DCE'd in the final binary.
+// TODO: Should be done with some sort of annotation on the declaration.
+void EnsureDeclarationsEmitted() {
+    ensureUsed(AllocInstance);
+    ensureUsed(AllocArrayInstance);
+    ensureUsed(InitInstance);
+    ensureUsed(InitSharedInstance);
+    ensureUsed(UpdateHeapRef);
+    ensureUsed(UpdateStackRef);
+    ensureUsed(UpdateReturnRef);
+    ensureUsed(ZeroHeapRef);
+    ensureUsed(ZeroArrayRefs);
+    ensureUsed(EnterFrame);
+    ensureUsed(LeaveFrame);
+    ensureUsed(AddTLSRecord);
+    ensureUsed(ClearTLSRecord);
+    ensureUsed(LookupTLS);
+    ensureUsed(MutationCheck);
+    ensureUsed(CheckLifetimesConstraint);
+    ensureUsed(FreezeSubgraph);
+    ensureUsed(FreezeSubgraph);
+}

runtime/src/main/cpp/Memory.h

@@ -21,55 +21,7 @@
 #include "Common.h"
 #include "TypeInfo.h"
 #include "Atomic.h"
-
+#include "PointerBits.h"
-typedef enum {
-  // Those bit masks are applied to refCount_ field.
-  // Container is normal thread-local container.
-  CONTAINER_TAG_LOCAL = 0,
-  // Container is frozen, could only refer to other frozen objects.
-  // Refcounter update is atomics.
-  CONTAINER_TAG_FROZEN = 1 | 1,  // shareable
-  // Stack container, no need to free, children cleanup still shall be there.
-  CONTAINER_TAG_STACK = 2,
-  // Atomic container, reference counter is atomically updated.
-  CONTAINER_TAG_SHARED = 3 | 1,  // shareable
-  // Shift to get actual counter.
-  CONTAINER_TAG_SHIFT = 2,
-  // Actual value to increment/decrement container by. Tag is in lower bits.
-  CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
-  // Mask for container type.
-  CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
-
-  // Shift to get actual object count, if has it.
-  CONTAINER_TAG_GC_SHIFT     = 7,
-  CONTAINER_TAG_GC_MASK      = (1 << CONTAINER_TAG_GC_SHIFT) - 1,
-  CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT,
-  // Color mask of a container.
-  CONTAINER_TAG_COLOR_SHIFT   = 3,
-  CONTAINER_TAG_GC_COLOR_MASK = (1 << CONTAINER_TAG_COLOR_SHIFT) - 1,
-  // Colors.
-  // In use or free.
-  CONTAINER_TAG_GC_BLACK  = 0,
-  // Possible member of garbage cycle.
-  CONTAINER_TAG_GC_GRAY   = 1,
-  // Member of garbage cycle.
-  CONTAINER_TAG_GC_WHITE  = 2,
-  // Possible root of cycle.
-  CONTAINER_TAG_GC_PURPLE = 3,
-  // Acyclic.
-  CONTAINER_TAG_GC_GREEN  = 4,
-  // Orange and red are currently unused.
-  // Candidate cycle awaiting epoch.
-  CONTAINER_TAG_GC_ORANGE = 5,
-  // Candidate cycle awaiting sigma computation.
-  CONTAINER_TAG_GC_RED    = 6,
-  // Individual state bits used during GC and freezing.
-  CONTAINER_TAG_GC_MARKED   = 1 << CONTAINER_TAG_COLOR_SHIFT,
-  CONTAINER_TAG_GC_BUFFERED = 1 << (CONTAINER_TAG_COLOR_SHIFT + 1),
-  CONTAINER_TAG_GC_SEEN     = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2),
-  // If indeed has more that one object.
-  CONTAINER_TAG_GC_HAS_OBJECT_COUNT = 1 << (CONTAINER_TAG_COLOR_SHIFT + 3)
-} ContainerTag;
 
 typedef enum {
   // Must match to permTag() in Kotlin.
@@ -79,258 +31,9 @@ typedef enum {
   OBJECT_TAG_MASK = (1 << 2) - 1
 } ObjectTag;
 
-typedef uint32_t container_size_t;
-
-// Header of all container objects. Contains reference counter.
-struct ContainerHeader {
-  // Reference counter of container. Uses CONTAINER_TAG_SHIFT, lower bits of counter
-  // for container type (for polymorphism in ::Release()).
-  uint32_t refCount_;
-  // Number of objects in the container.
-  uint32_t objectCount_;
-
-  inline bool local() const {
-      return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_LOCAL;
-  }
-
-  inline bool frozen() const {
-    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN;
-  }
-
-  inline void freeze() {
-    refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_FROZEN;
-  }
-
-  inline void makeShared() {
-      refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_SHARED;
-  }
-
-  inline bool shared() const {
-    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_SHARED;
-  }
-
-  inline bool shareable() const {
-      return (tag() & 1) != 0; // CONTAINER_TAG_FROZEN || CONTAINER_TAG_SHARED
-  }
-
-  inline bool stack() const {
-    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK;
-  }
-
-  inline int refCount() const {
-    return (int)refCount_ >> CONTAINER_TAG_SHIFT;
-  }
-
-  inline void setRefCount(unsigned refCount) {
-    refCount_ = tag() | (refCount << CONTAINER_TAG_SHIFT);
-  }
-
-  template <bool Atomic>
-  inline void incRefCount() {
-#ifdef KONAN_NO_THREADS
-    refCount_ += CONTAINER_TAG_INCREMENT;
-#else
-    if (Atomic)
-      __sync_add_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT);
-    else
-      refCount_ += CONTAINER_TAG_INCREMENT;
-#endif
-  }
-
-  template <bool Atomic>
-  inline bool tryIncRefCount() {
-    if (Atomic) {
-      while (true) {
-        uint32_t currentRefCount_ = refCount_;
-        if (((int)currentRefCount_ >> CONTAINER_TAG_SHIFT) > 0) {
-          if (compareAndSet(&refCount_, currentRefCount_, currentRefCount_ + CONTAINER_TAG_INCREMENT)) {
-            return true;
-          }
-        } else {
-          return false;
-        }
-      }
-    } else {
-      // Note: tricky case here is doing this during cycle collection.
-      // This can actually happen due to deallocation hooks.
-      // Fortunately by this point reference counts have been made precise again.
-      if (refCount() > 0) {
-        incRefCount</* Atomic = */ false>();
-        return true;
-      } else {
-        return false;
-      }
-    }
-  }
-
-  template <bool Atomic>
-  inline int decRefCount() {
-#ifdef KONAN_NO_THREADS
-    int value = refCount_ -= CONTAINER_TAG_INCREMENT;
-#else
-    int value = Atomic ?
-       __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT;
-#endif
-    return value >> CONTAINER_TAG_SHIFT;
-  }
-
-  inline int decRefCount() {
-  #ifdef KONAN_NO_THREADS
-      int value = refCount_ -= CONTAINER_TAG_INCREMENT;
-  #else
-      int value = shareable() ?
-         __sync_sub_and_fetch(&refCount_, CONTAINER_TAG_INCREMENT) : refCount_ -= CONTAINER_TAG_INCREMENT;
-  #endif
-      return value >> CONTAINER_TAG_SHIFT;
-  }
-
-  inline unsigned tag() const {
-    return refCount_ & CONTAINER_TAG_MASK;
-  }
-
-  inline unsigned objectCount() const {
-    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0 ?
-        (objectCount_ >> CONTAINER_TAG_GC_SHIFT) : 1;
-  }
-
-  inline void incObjectCount() {
-    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) != 0, "Must have object count");
-    objectCount_ += CONTAINER_TAG_GC_INCREMENT;
-  }
-
-  inline void setObjectCount(int count) {
-    if (count == 1) {
-      objectCount_ &= ~CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
-    } else {
-      objectCount_ = (count << CONTAINER_TAG_GC_SHIFT) | CONTAINER_TAG_GC_HAS_OBJECT_COUNT;
-    }
-  }
-
-  inline unsigned containerSize() const {
-    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must be single-object");
-    return (objectCount_ >> CONTAINER_TAG_GC_SHIFT);
-  }
-
-  inline void setContainerSize(unsigned size) {
-    RuntimeAssert((objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0, "Must not have object count");
-    objectCount_ = (objectCount_ & CONTAINER_TAG_GC_MASK) | (size << CONTAINER_TAG_GC_SHIFT);
-  }
-
-  inline bool hasContainerSize() {
-    return (objectCount_ & CONTAINER_TAG_GC_HAS_OBJECT_COUNT) == 0;
-  }
-
-  inline unsigned color() const {
-    return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK;
-  }
-
-  inline void setColorAssertIfGreen(unsigned color) {
-    RuntimeAssert(this->color() != CONTAINER_TAG_GC_GREEN, "Must not be green");
-    setColorEvenIfGreen(color);
-  }
-
-  inline void setColorEvenIfGreen(unsigned color) {
-    // TODO: do we need atomic color update?
-    objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
-  }
-
-  inline void setColorUnlessGreen(unsigned color) {
-    // TODO: do we need atomic color update?
-    unsigned objectCount = objectCount_;
-    if ((objectCount & CONTAINER_TAG_GC_COLOR_MASK) != CONTAINER_TAG_GC_GREEN)
-        objectCount_ = (objectCount & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
-  }
-
-  inline bool buffered() const {
-    return (objectCount_ & CONTAINER_TAG_GC_BUFFERED) != 0;
-  }
-
-  inline void setBuffered() {
-    objectCount_ |= CONTAINER_TAG_GC_BUFFERED;
-  }
-
-  inline void resetBuffered() {
-    objectCount_ &= ~CONTAINER_TAG_GC_BUFFERED;
-  }
-
-  inline bool marked() const {
-    return (objectCount_ & CONTAINER_TAG_GC_MARKED) != 0;
-  }
-
-  inline void mark() {
-    objectCount_ |= CONTAINER_TAG_GC_MARKED;
-  }
-
-  inline void unMark() {
-    objectCount_ &= ~CONTAINER_TAG_GC_MARKED;
-  }
-
-  inline bool seen() const {
-    return (objectCount_ & CONTAINER_TAG_GC_SEEN) != 0;
-  }
-
-  inline void setSeen() {
-    objectCount_ |= CONTAINER_TAG_GC_SEEN;
-  }
-
-  inline void resetSeen() {
-    objectCount_ &= ~CONTAINER_TAG_GC_SEEN;
-  }
-
-  // Following operations only work on freed container which is in finalization queue.
-  // We cannot use 'this' here, as it conflicts with aliasing analysis in clang.
-  inline void setNextLink(ContainerHeader* next) {
-    *reinterpret_cast<ContainerHeader**>(this + 1) = next;
-  }
-
-  inline ContainerHeader* nextLink() {
-    return *reinterpret_cast<ContainerHeader**>(this + 1);
-  }
-};
-
 struct ArrayHeader;
 struct MetaObjHeader;
 
-template <typename T>
-ALWAYS_INLINE T* setPointerBits(T* ptr, unsigned bits) {
-  return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) | bits);
-}
-
-template <typename T>
-ALWAYS_INLINE T* clearPointerBits(T* ptr, unsigned bits) {
-  return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) & ~static_cast<uintptr_t>(bits));
-}
-
-template <typename T>
-ALWAYS_INLINE unsigned getPointerBits(T* ptr, unsigned bits) {
-  return reinterpret_cast<uintptr_t>(ptr) & static_cast<uintptr_t>(bits);
-}
-
-template <typename T>
-ALWAYS_INLINE bool hasPointerBits(T* ptr, unsigned bits) {
-  return getPointerBits(ptr, bits) != 0;
-}
-
-// Header for the meta-object.
-struct MetaObjHeader {
-  // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
-  const TypeInfo* typeInfo_;
-  // Container pointer.
-  ContainerHeader* container_;
-
-#ifdef KONAN_OBJC_INTEROP
-  void* associatedObject_;
-#endif
-
-  // Flags for the object state.
-  int32_t flags_;
-
-  struct {
-    // Strong reference to the counter object.
-    ObjHeader* counter_;
-  } WeakReference;
-};
-
 // Header of every object.
 struct ObjHeader {
   TypeInfo* typeInfoOrMeta_;
@@ -350,19 +53,13 @@ struct ObjHeader {
         createMetaObject(&typeInfoOrMeta_);
   }
 
-  void setContainer(ContainerHeader* container) {
+  ALWAYS_INLINE ObjHeader** GetWeakCounterLocation();
-    meta_object()->container_ = container;
-    typeInfoOrMeta_ = setPointerBits(typeInfoOrMeta_, OBJECT_TAG_NONTRIVIAL_CONTAINER);
-  }
 
-  ContainerHeader* container() const {
+#ifdef KONAN_OBJC_INTEROP
-    unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
+  ALWAYS_INLINE void* GetAssociatedObject();
-    if ((bits & (OBJECT_TAG_PERMANENT_CONTAINER | OBJECT_TAG_NONTRIVIAL_CONTAINER)) == 0)
+  ALWAYS_INLINE void** GetAssociatedObjectLocation();
-      return reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(this)) - 1;
+  ALWAYS_INLINE void SetAssociatedObject(void* obj);
-    if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0)
+#endif
-      return nullptr;
-    return (reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_;
-  }
 
   inline bool local() const {
     unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
@@ -397,10 +94,12 @@ struct ArrayHeader {
   uint32_t count_;
 };
 
-inline bool isPermanentOrFrozen(ObjHeader* obj) {
+ALWAYS_INLINE bool isFrozen(const ObjHeader* obj);
-    auto* container = obj->container();
+ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj);
-    return container == nullptr || container->frozen();
+ALWAYS_INLINE bool isShareable(const ObjHeader* obj);
-}
+
+class ForeignRefManager;
+typedef ForeignRefManager* ForeignRefContext;
 
 #ifdef __cplusplus
 extern "C" {
@@ -409,10 +108,6 @@ extern "C" {
 #define OBJ_RESULT __result__
 #define OBJ_GETTER0(name) ObjHeader* name(ObjHeader** OBJ_RESULT)
 #define OBJ_GETTER(name, ...) ObjHeader* name(__VA_ARGS__, ObjHeader** OBJ_RESULT)
-#define MODEL_VARIANTS(returnType, name, ...)            \
-   returnType name(__VA_ARGS__) RUNTIME_NOTHROW;         \
-   returnType name##Strict(__VA_ARGS__) RUNTIME_NOTHROW; \
-   returnType name##Relaxed(__VA_ARGS__) RUNTIME_NOTHROW;
 #define RETURN_OBJ(value) { ObjHeader* __obj = value; \
     UpdateReturnRef(OBJ_RESULT, __obj);               \
     return __obj; }
@@ -443,32 +138,16 @@ void RestoreMemory(MemoryState*);
 // Escape analysis algorithm is the provider of information for decision on exact aux slot
 // selection, and comes from upper bound esteemation of object lifetime.
 //
-OBJ_GETTER(AllocInstanceStrict, const TypeInfo* type_info) RUNTIME_NOTHROW;
-OBJ_GETTER(AllocInstanceRelaxed, const TypeInfo* type_info) RUNTIME_NOTHROW;
 OBJ_GETTER(AllocInstance, const TypeInfo* type_info) RUNTIME_NOTHROW;
 
-OBJ_GETTER(AllocArrayInstanceStrict, const TypeInfo* type_info, int32_t elements);
-OBJ_GETTER(AllocArrayInstanceRelaxed, const TypeInfo* type_info, int32_t elements);
 OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements);
 
-OBJ_GETTER(InitInstanceStrict,
-    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
-OBJ_GETTER(InitInstanceRelaxed,
-    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
 OBJ_GETTER(InitInstance,
     ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
 
-OBJ_GETTER(InitSharedInstanceStrict,
-    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
-OBJ_GETTER(InitSharedInstanceRelaxed,
-    ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
 OBJ_GETTER(InitSharedInstance,
     ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*));
 
-// Weak reference operations.
-// Atomically clears counter object reference.
-void WeakReferenceCounterClear(ObjHeader* counter);
-
 //
 // Object reference management.
 //
@@ -495,23 +174,23 @@ void WeakReferenceCounterClear(ObjHeader* counter);
 extern const bool IsStrictMemoryModel;
 
 // Sets stack location.
-MODEL_VARIANTS(void, SetStackRef, ObjHeader** location, const ObjHeader* object);
+void SetStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Sets heap location.
-MODEL_VARIANTS(void, SetHeapRef, ObjHeader** location, const ObjHeader* object);
+void SetHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Zeroes heap location.
 void ZeroHeapRef(ObjHeader** location) RUNTIME_NOTHROW;
 // Zeroes an array.
 void ZeroArrayRefs(ArrayHeader* array) RUNTIME_NOTHROW;
 // Zeroes stack location.
-MODEL_VARIANTS(void, ZeroStackRef, ObjHeader** location);
+void ZeroStackRef(ObjHeader** location) RUNTIME_NOTHROW;
 // Updates stack location.
-MODEL_VARIANTS(void, UpdateStackRef, ObjHeader** location, const ObjHeader* object);
+void UpdateStackRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates heap/static data location.
-MODEL_VARIANTS(void, UpdateHeapRef, ObjHeader** location, const ObjHeader* object);
+void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates location if it is null, atomically.
-MODEL_VARIANTS(void, UpdateHeapRefIfNull, ObjHeader** location, const ObjHeader* object);
+void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) RUNTIME_NOTHROW;
 // Updates reference in return slot.
-MODEL_VARIANTS(void, UpdateReturnRef, ObjHeader** returnSlot, const ObjHeader* object);
+void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) RUNTIME_NOTHROW;
 // Compares and swaps reference with taken lock.
 OBJ_GETTER(SwapHeapRefLocked,
     ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock,
@@ -522,9 +201,9 @@ void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlo
 // Reads reference with taken lock.
 OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) RUNTIME_NOTHROW;
 // Called on frame enter, if it has object slots.
-MODEL_VARIANTS(void, EnterFrame, ObjHeader** start, int parameters, int count);
+void EnterFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
 // Called on frame leave, if it has object slots.
-MODEL_VARIANTS(void, LeaveFrame, ObjHeader** start, int parameters, int count);
+void LeaveFrame(ObjHeader** start, int parameters, int count) RUNTIME_NOTHROW;
 // Clears object subgraph references from memory subsystem, and optionally
 // checks if subgraph referenced by given root is disjoint from the rest of
 // object graph, i.e. no external references exists.
@@ -559,11 +238,26 @@ void GC_CollectorCallback(void* worker) RUNTIME_NOTHROW;
 bool Kotlin_Any_isShareable(ObjHeader* thiz);
 void PerformFullGC() RUNTIME_NOTHROW;
 
+bool TryAddHeapRef(const ObjHeader* object);
+
+void ReleaseHeapRef(const ObjHeader* object) RUNTIME_NOTHROW;
+void ReleaseHeapRefNoCollect(const ObjHeader* object) RUNTIME_NOTHROW;
+
+ForeignRefContext InitLocalForeignRef(ObjHeader* object);
+
+ForeignRefContext InitForeignRef(ObjHeader* object);
+void DeinitForeignRef(ObjHeader* object, ForeignRefContext context);
+
+bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context);
+
+// Should be used when reference is read from a possibly shared variable,
+// and there's nothing else keeping the object alive.
+void AdoptReferenceFromSharedVariable(ObjHeader* object);
+
 #ifdef __cplusplus
 }
 #endif
 
-
 struct FrameOverlay {
   void* arena;
   FrameOverlay* previous;
@@ -624,7 +318,4 @@ class ExceptionObjHolder {
    ObjHeader* obj_;
 };
 
-class ForeignRefManager;
-typedef ForeignRefManager* ForeignRefContext;
-
 #endif // RUNTIME_MEMORY_H

runtime/src/main/cpp/MemoryPrivate.hpp

@@ -1,44 +0,0 @@
-/*
- * Copyright 2010-2017 JetBrains s.r.o.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#ifndef RUNTIME_MEMORYPRIVATE_HPP
-#define RUNTIME_MEMORYPRIVATE_HPP
-
-#include "Memory.h"
-
-extern "C" {
-
-bool TryAddHeapRef(const ObjHeader* object);
-
-MODEL_VARIANTS(void, ReleaseHeapRef, const ObjHeader* object);
-MODEL_VARIANTS(void, ReleaseHeapRefNoCollect, const ObjHeader* object);
-
-void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
-
-ForeignRefContext InitLocalForeignRef(ObjHeader* object);
-
-ForeignRefContext InitForeignRef(ObjHeader* object);
-void DeinitForeignRef(ObjHeader* object, ForeignRefContext context);
-
-bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context);
-
-// Should be used when reference is read from a possibly shared variable,
-// and there's nothing else keeping the object alive.
-void AdoptReferenceFromSharedVariable(ObjHeader* object);
-
-}  // extern "C"
-
-#endif // RUNTIME_MEMORYPRIVATE_HPP

runtime/src/main/cpp/MemorySharedRefs.cpp

@@ -4,7 +4,6 @@
  */
 
 #include "Exceptions.h"
-#include "MemoryPrivate.hpp"
 #include "MemorySharedRefs.hpp"
 #include "Runtime.h"
 #include "Types.h"

runtime/src/main/cpp/ObjCExport.h

@@ -14,17 +14,17 @@ extern "C" id objc_retainBlock(id self);
 extern "C" void objc_release(id self);
 
 inline static id GetAssociatedObject(ObjHeader* obj) {
-  return (id)obj->meta_object()->associatedObject_;
+    return (id)obj->GetAssociatedObject();
 }
 
 // Note: this function shall not be used on shared objects.
 inline static void SetAssociatedObject(ObjHeader* obj, id value) {
-  obj->meta_object()->associatedObject_ = (void*)value;
+    obj->SetAssociatedObject((void*)value);
 }
 
 inline static id AtomicCompareAndSwapAssociatedObject(ObjHeader* obj, id expectedValue, id newValue) {
-  id* location = reinterpret_cast<id*>(&obj->meta_object()->associatedObject_);
+    id* location = reinterpret_cast<id*>(obj->GetAssociatedObjectLocation());
-  return __sync_val_compare_and_swap(location, expectedValue, newValue);
+    return __sync_val_compare_and_swap(location, expectedValue, newValue);
 }
 
 inline static OBJ_GETTER(AllocInstanceWithAssociatedObject, const TypeInfo* typeInfo, id associatedObject) {

runtime/src/main/cpp/ObjCExport.mm

@@ -37,7 +37,7 @@
 #import "ObjCExport.h"
 #import "ObjCExportInit.h"
 #import "ObjCExportPrivate.h"
-#import "MemoryPrivate.hpp"
+#import "ObjCMMAPI.h"
 #import "Runtime.h"
 #import "Utils.h"
 #import "Exceptions.h"
@@ -122,7 +122,6 @@ extern "C" OBJ_GETTER(Kotlin_ObjCExport_AllocInstanceWithAssociatedObject,
 
 static Class getOrCreateClass(const TypeInfo* typeInfo);
 static void initializeClass(Class clazz);
-extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
 
 extern "C" id objc_retainAutoreleaseReturnValue(id self);
 
@@ -158,7 +157,7 @@ extern "C" id Kotlin_ObjCExport_CreateNSStringFromKString(ObjHeader* str) {
       length:numBytes
       encoding:NSUTF16LittleEndianStringEncoding];
 
-    if (!str->container()->shareable()) {
+    if (!isShareable(str)) {
       SetAssociatedObject(str, candidate);
     } else {
       id old = AtomicCompareAndSwapAssociatedObject(str, nullptr, candidate);
@@ -485,11 +484,9 @@ template <bool retainAutorelease>
 static ALWAYS_INLINE id Kotlin_ObjCExport_refToObjCImpl(ObjHeader* obj) {
   if (obj == nullptr) return nullptr;
 
-  if (obj->has_meta_object()) {
+  id associatedObject = GetAssociatedObject(obj);
-    id associatedObject = GetAssociatedObject(obj);
+  if (associatedObject != nullptr) {
-    if (associatedObject != nullptr) {
+    return retainAutorelease ? objc_retainAutoreleaseReturnValue(associatedObject) : associatedObject;
-      return retainAutorelease ? objc_retainAutoreleaseReturnValue(associatedObject) : associatedObject;
-    }
   }
 
   // TODO: propagate [retainAutorelease] to the code below.

runtime/src/main/cpp/ObjCInterop.mm

@@ -156,11 +156,10 @@ static inline void SetKotlinTypeInfo(Class clazz, const TypeInfo* typeInfo) {
 const TypeInfo* GetObjCKotlinTypeInfo(ObjHeader* obj) RUNTIME_NOTHROW;
 
 RUNTIME_NOTHROW const TypeInfo* GetObjCKotlinTypeInfo(ObjHeader* obj) {
-  RuntimeAssert(obj->has_meta_object(), "");
+    void* objcPtr = obj->GetAssociatedObject();
-  void* objcPtr = obj->meta_object()->associatedObject_;
+    RuntimeAssert(objcPtr != nullptr, "");
-  RuntimeAssert(objcPtr != nullptr, "");
+    Class clazz = object_getClass(reinterpret_cast<id>(objcPtr));
-  Class clazz = object_getClass(reinterpret_cast<id>(objcPtr));
+    return GetKotlinClassData(clazz)->typeInfo;
-  return GetKotlinClassData(clazz)->typeInfo;
 }
 
 

runtime/src/main/cpp/ObjCInteropUtils.mm

@@ -55,11 +55,8 @@ id Kotlin_Interop_CreateNSStringFromKString(ObjHeader* str) {
     return nullptr;
   }
 
-  if (str->has_meta_object()) {
+  if (void* associatedObject = str->GetAssociatedObject()) {
-    void* associatedObject = str->meta_object()->associatedObject_;
+    return (id)associatedObject;
-    if (associatedObject != nullptr) {
-      return (id)associatedObject;
-    }
   }
 
   return Kotlin_ObjCExport_CreateNSStringFromKString(str);
@@ -79,7 +76,7 @@ OBJ_GETTER(Kotlin_Interop_CreateKStringFromNSString, NSString* str) {
 
   CFStringGetCharacters(immutableCopyOrSameStr, range, rawResult);
 
-  result->obj()->meta_object()->associatedObject_ = (void*)immutableCopyOrSameStr;
+  result->obj()->SetAssociatedObject((void*)immutableCopyOrSameStr);
 
   RETURN_OBJ(result->obj());
 }

runtime/src/main/cpp/ObjCMMAPI.h

@@ -0,0 +1,17 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_OBJCMMAPI_H
+#define RUNTIME_OBJCMMAPI_H
+
+#include "Common.h"
+
+#if KONAN_OBJC_INTEROP
+
+extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
+
+#endif // KONAN_OBJC_INTEROP
+
+#endif // RUNTIME_OBJCMMAPI_H

runtime/src/main/cpp/PointerBits.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_POINTER_BITS_H
+#define RUNTIME_POINTER_BITS_H
+
+#include <cstdint>
+
+#include "Common.h"
+
+template <typename T>
+ALWAYS_INLINE T* setPointerBits(T* ptr, unsigned bits) {
+    return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) | bits);
+}
+
+template <typename T>
+ALWAYS_INLINE T* clearPointerBits(T* ptr, unsigned bits) {
+    return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) & ~static_cast<uintptr_t>(bits));
+}
+
+template <typename T>
+ALWAYS_INLINE unsigned getPointerBits(T* ptr, unsigned bits) {
+    return reinterpret_cast<uintptr_t>(ptr) & static_cast<uintptr_t>(bits);
+}
+
+template <typename T>
+ALWAYS_INLINE bool hasPointerBits(T* ptr, unsigned bits) {
+    return getPointerBits(ptr, bits) != 0;
+}
+
+#endif // RUNTIME_POINTER_BITS_H

runtime/src/main/cpp/Weak.cpp

@@ -13,6 +13,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+
+#include "Weak.h"
+
 #include "Memory.h"
 #include "Types.h"
 
@@ -55,25 +58,24 @@ OBJ_GETTER(makePermanentWeakReferenceImpl, ObjHeader*);
 // See Weak.kt for implementation details.
 // Retrieve link on the counter object.
 OBJ_GETTER(Konan_getWeakReferenceImpl, ObjHeader* referred) {
-  if (referred->container() == nullptr) {
+    if (referred->permanent()) {
-    RETURN_RESULT_OF(makePermanentWeakReferenceImpl, referred);
+        RETURN_RESULT_OF(makePermanentWeakReferenceImpl, referred);
-  }
+    }
-
-  MetaObjHeader* meta = referred->meta_object();
 
 #if KONAN_OBJC_INTEROP
   if (IsInstance(referred, theObjCObjectWrapperTypeInfo)) {
-    RETURN_RESULT_OF(makeObjCWeakReferenceImpl, meta->associatedObject_);
+      RETURN_RESULT_OF(makeObjCWeakReferenceImpl, referred->GetAssociatedObject());
   }
 #endif // KONAN_OBJC_INTEROP
 
-  if (meta->WeakReference.counter_ == nullptr) {
+  ObjHeader** weakCounterLocation = referred->GetWeakCounterLocation();
-     ObjHolder counterHolder;
+  if (*weakCounterLocation == nullptr) {
-     // Cast unneeded, just to emphasize we store an object reference as void*.
+      ObjHolder counterHolder;
-     ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast<void*>(referred), counterHolder.slot());
+      // Cast unneeded, just to emphasize we store an object reference as void*.
-     UpdateHeapRefIfNull(&meta->WeakReference.counter_, counter);
+      ObjHeader* counter = makeWeakReferenceCounter(reinterpret_cast<void*>(referred), counterHolder.slot());
+      UpdateHeapRefIfNull(weakCounterLocation, counter);
   }
-  RETURN_OBJ(meta->WeakReference.counter_);
+  RETURN_OBJ(*weakCounterLocation);
 }
 
 // Materialize a weak reference to either null or the real reference.

runtime/src/main/cpp/Weak.h

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#ifndef RUNTIME_WEAK_H
+#define RUNTIME_WEAK_H
+
+#include "Memory.h"
+
+extern "C" {
+
+// Atomically clears counter object reference.
+void WeakReferenceCounterClear(ObjHeader* counter);
+
+} // extern "C"
+
+#endif // RUNTIME_WEAK_H

runtime/src/mm/cpp/Stubs.cpp

@@ -0,0 +1,237 @@
+/*
+ * Copyright 2010-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license
+ * that can be found in the LICENSE file.
+ */
+
+#include "Memory.h"
+
+ALWAYS_INLINE bool isFrozen(const ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+ALWAYS_INLINE bool isPermanentOrFrozen(const ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+ALWAYS_INLINE bool isShareable(const ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+ObjHeader** ObjHeader::GetWeakCounterLocation() {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+#ifdef KONAN_OBJC_INTEROP
+
+void* ObjHeader::GetAssociatedObject() {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void** ObjHeader::GetAssociatedObjectLocation() {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void ObjHeader::SetAssociatedObject(void* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+#endif // KONAN_OBJC_INTEROP
+
+static MetaObjHeader* createMetaObject(TypeInfo** location) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+static void destroyMetaObject(TypeInfo** location) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+extern "C" {
+
+MemoryState* InitMemory() {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void DeinitMemory(MemoryState*) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void RestoreMemory(MemoryState* memoryState) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW OBJ_GETTER(AllocInstance, const TypeInfo* type_info) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+OBJ_GETTER(AllocArrayInstance, const TypeInfo* type_info, int32_t elements) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+OBJ_GETTER(InitInstance, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+OBJ_GETTER(InitSharedInstance, ObjHeader** location, const TypeInfo* typeInfo, void (*ctor)(ObjHeader*)) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+extern const bool IsStrictMemoryModel = true;
+
+RUNTIME_NOTHROW void SetStackRef(ObjHeader** location, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void SetHeapRef(ObjHeader** location, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ZeroHeapRef(ObjHeader** location) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ZeroArrayRefs(ArrayHeader* array) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ZeroStackRef(ObjHeader** location) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void UpdateHeapRef(ObjHeader** location, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void UpdateHeapRefIfNull(ObjHeader** location, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW OBJ_GETTER(
+        SwapHeapRefLocked, ObjHeader** location, ObjHeader* expectedValue, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void SetHeapRefLocked(ObjHeader** location, ObjHeader* newValue, int32_t* spinlock, int32_t* cookie) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW OBJ_GETTER(ReadHeapRefLocked, ObjHeader** location, int32_t* spinlock, int32_t* cookie) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void EnterFrame(ObjHeader** start, int parameters, int count) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void LeaveFrame(ObjHeader** start, int parameters, int count) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void* CreateStablePointer(ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void DisposeStablePointer(void* pointer) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW OBJ_GETTER(DerefStablePointer, void*) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW OBJ_GETTER(AdoptStablePointer, void*) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void MutationCheck(ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void CheckLifetimesConstraint(ObjHeader* obj, ObjHeader* pointee) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void FreezeSubgraph(ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void EnsureNeverFrozen(ObjHeader* obj) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void AddTLSRecord(MemoryState* memory, void** key, int size) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ClearTLSRecord(MemoryState* memory, void** key) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW ObjHeader** LookupTLS(void** key, int index) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void GC_RegisterWorker(void* worker) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void GC_UnregisterWorker(void* worker) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void GC_CollectorCallback(void* worker) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+bool Kotlin_Any_isShareable(ObjHeader* thiz) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void PerformFullGC() {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+bool TryAddHeapRef(const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ReleaseHeapRef(const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+RUNTIME_NOTHROW void ReleaseHeapRefNoCollect(const ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+ForeignRefContext InitLocalForeignRef(ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+ForeignRefContext InitForeignRef(ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void DeinitForeignRef(ObjHeader* object, ForeignRefContext context) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+bool IsForeignRefAccessible(ObjHeader* object, ForeignRefContext context) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+void AdoptReferenceFromSharedVariable(ObjHeader* object) {
+    RuntimeCheck(false, "Unimplemented");
+}
+
+} // extern "C"

runtime/src/objc/cpp/ObjCExportClasses.mm

@@ -21,7 +21,6 @@
 #import "ObjCExport.h"
 #import "ObjCExportInit.h"
 #import "ObjCExportPrivate.h"
-#import "MemoryPrivate.hpp"
 #import "Runtime.h"
 #import "Utils.h"
 #import "Exceptions.h"
@@ -92,7 +91,7 @@ static void injectToRuntime();
   candidate->permanent = obj->permanent();
 
   if (!obj->permanent()) { // TODO: permanent objects should probably be supported as custom types.
-    if (!obj->container()->shareable()) {
+    if (!isShareable(obj)) {
       SetAssociatedObject(obj, candidate);
     } else {
       id old = AtomicCompareAndSwapAssociatedObject(obj, nullptr, candidate);

runtime/src/objc/cpp/ObjCInteropUtilsClasses.mm

@@ -83,8 +83,7 @@ void objc_release(id obj);
 extern "C" OBJ_GETTER(Kotlin_Interop_refFromObjC, id obj);
 
 static OBJ_GETTER(Konan_ObjCInterop_getWeakReference, KRef ref) {
-  MetaObjHeader* meta = ref->meta_object();
+  KotlinObjCWeakReference* objcRef = (KotlinObjCWeakReference*)ref->GetAssociatedObject();
-  KotlinObjCWeakReference* objcRef = (KotlinObjCWeakReference*)meta->associatedObject_;
 
   id objcReferred = objc_loadWeakRetained(&objcRef->referred);
   KRef result = Kotlin_Interop_refFromObjC(objcReferred, OBJ_RESULT);
@@ -94,10 +93,9 @@ static OBJ_GETTER(Konan_ObjCInterop_getWeakReference, KRef ref) {
 }
 
 static void Konan_ObjCInterop_initWeakReference(KRef ref, id objcPtr) {
-  MetaObjHeader* meta = ref->meta_object();
   KotlinObjCWeakReference* objcRef = [KotlinObjCWeakReference new];
   objc_storeWeak(&objcRef->referred, objcPtr);
-  meta->associatedObject_ = objcRef;
+  ref->SetAssociatedObject(objcRef);
 }
 
 __attribute__((constructor))

runtime/src/relaxed/cpp/MemoryImpl.cpp

@@ -3,7 +3,7 @@
  * that can be found in the LICENSE file.
  */
 #include "Memory.h"
-#include "MemoryPrivate.hpp"
+#include "../../legacymm/cpp/MemoryPrivate.hpp" // Fine, because this module is a part of legacy MM.
 
 // Note that only C++ part of the runtime goes via those functions, Kotlin uses specialized versions.
 
@@ -65,4 +65,8 @@ void LeaveFrame(ObjHeader** start, int parameters, int count) {
   LeaveFrameRelaxed(start, parameters, count);
 }
 
+void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
+    UpdateStackRefRelaxed(location, object);
+}
+
 }  // extern "C"

runtime/src/strict/cpp/MemoryImpl.cpp

@@ -3,7 +3,7 @@
  * that can be found in the LICENSE file.
  */
 #include "Memory.h"
-#include "MemoryPrivate.hpp"
+#include "../../legacymm/cpp/MemoryPrivate.hpp" // Fine, because this module is a part of legacy MM.
 
 // Note that only C++ part of the runtime goes via those functions, Kotlin uses specialized versions.
 
@@ -65,4 +65,8 @@ void LeaveFrame(ObjHeader** start, int parameters, int count) {
   LeaveFrameStrict(start, parameters, count);
 }
 
+void UpdateStackRef(ObjHeader** location, const ObjHeader* object) {
+    UpdateStackRefStrict(location, object);
+}
+
 }  // extern "C"

shared/src/main/kotlin/org/jetbrains/kotlin/konan/target/KonanTargetExtenstions.kt

@@ -1,5 +1,7 @@
 package org.jetbrains.kotlin.konan.target
 
+// TODO: This all needs to go to konan.properties
+
 fun KonanTarget.supportsCodeCoverage(): Boolean =
         this == KonanTarget.MINGW_X64 ||
         this == KonanTarget.LINUX_X64 ||
@@ -21,3 +23,10 @@ fun KonanTarget.supportsMimallocAllocator(): Boolean =
         is KonanTarget.IOS_X64 -> true
         else -> false // watchOS/tvOS/android_x86/android_arm32 aren't tested; linux_mips32/linux_mipsel32 need linking with libatomic.
     }
+
+fun KonanTarget.supportsThreads(): Boolean =
+     when(this) {
+        is KonanTarget.WASM32 -> false
+        is KonanTarget.ZEPHYR -> false
+        else -> true
+     }