Reduce size of object header by one pointer. (#2432)

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt

@@ -9,7 +9,6 @@ package org.jetbrains.kotlin.backend.konan.llvm
 import kotlinx.cinterop.*
 import llvm.*
 import org.jetbrains.kotlin.backend.konan.*
-import org.jetbrains.kotlin.backend.konan.descriptors.*
 import org.jetbrains.kotlin.backend.konan.descriptors.isInterface
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.ClassConstructorDescriptor
@@ -28,6 +27,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
     fun llvmFunction(function: FunctionDescriptor): LLVMValueRef = function.llvmFunction
     val intPtrType = LLVMIntPtrType(llvmTargetData)!!
     internal val immOneIntPtrType = LLVMConstInt(intPtrType, 1, 1)!!
+    // Keep in sync with OBJECT_TAG_MASK in C++.
+    internal val immTypeInfoMask = LLVMConstNot(LLVMConstInt(intPtrType, 3, 0)!!)!!
 
     //-------------------------------------------------------------------------//
 
@@ -624,7 +625,11 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
             call(context.llvm.getObjCKotlinTypeInfo, listOf(receiver))
         } else {
             val typeInfoOrMetaPtr = structGep(receiver, 0  /* typeInfoOrMeta_ */)
-            val typeInfoOrMeta = load(typeInfoOrMetaPtr)
+            val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr)
+            // Clear two lower bits.
+            val typeInfoOrMetaWithFlagsRaw = ptrToInt(typeInfoOrMetaWithFlags, codegen.intPtrType)
+            val typeInfoOrMetaRaw = and(typeInfoOrMetaWithFlagsRaw, codegen.immTypeInfoMask)
+            val typeInfoOrMeta = intToPtr(typeInfoOrMetaRaw, kTypeInfoPtr)
             val typeInfoPtrPtr = structGep(typeInfoOrMeta, 0 /* typeInfo */)
             load(typeInfoPtrPtr)
         }
@@ -634,7 +639,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
         /*
          * Resolve owner of the call with special handling of Any methods:
          * if toString/eq/hc is invoked on an interface instance, we resolve
-         * owner as Any and dispatch it via vtable
+         * owner as Any and dispatch it via vtable.
          */
         val anyMethod = (descriptor as SimpleFunctionDescriptor).findOverriddenMethodOfAny()
         val owner = (anyMethod ?: descriptor).containingDeclaration as ClassDescriptor

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt

@@ -457,8 +457,6 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
             origin = context.standardLlvmSymbolsOrigin
     )
 
-    val staticContainer = importRtGlobal("theStaticObjectsContainer")
-
     val memsetFunction = importMemset()
 
     val usedFunctions = mutableListOf<LLVMValueRef>()

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt

@@ -5,10 +5,8 @@
 
 package org.jetbrains.kotlin.backend.konan.llvm
 
-import llvm.LLVMLinkage
-import llvm.LLVMSetLinkage
-import llvm.LLVMTypeRef
-import llvm.LLVMValueRef
+import kotlinx.cinterop.cValuesOf
+import llvm.*
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.fqNameSafe
 import org.jetbrains.kotlin.backend.konan.irasdescriptors.llvmSymbolOrigin
 import org.jetbrains.kotlin.descriptors.ClassDescriptor
@@ -17,16 +15,23 @@ import org.jetbrains.kotlin.ir.declarations.IrClass
 import org.jetbrains.kotlin.name.FqName
 import org.jetbrains.kotlin.name.Name
 
+private fun ConstPointer.add(index: Int): ConstPointer {
+    return constPointer(LLVMConstGEP(llvm, cValuesOf(Int32(index).llvm), 1)!!)
+}
+
+// Must match OBJECT_TAG_PERMANENT_CONTAINER in C++.
+private fun StaticData.permanentTag(typeInfo: ConstPointer): ConstPointer {
+    // Only pointer arithmetic via GEP works on constant pointers in LLVM.
+    return typeInfo.bitcast(int8TypePtr).add(1).bitcast(kTypeInfoPtr)
+}
 
 private fun StaticData.objHeader(typeInfo: ConstPointer): Struct {
-    val container = constValue(context.llvm.staticContainer)
-    return Struct(runtime.objHeaderType, typeInfo, container)
+    return Struct(runtime.objHeaderType, permanentTag(typeInfo))
 }
 
 private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct {
     assert (length >= 0)
-    val container = constValue(context.llvm.staticContainer)
-    return Struct(runtime.arrayHeaderType, typeInfo, container, Int32(length))
+    return Struct(runtime.arrayHeaderType, permanentTag(typeInfo), Int32(length))
 }
 
 internal fun StaticData.createKotlinStringLiteral(value: String): ConstPointer {

runtime/src/main/cpp/Atomic.cpp

@@ -168,7 +168,7 @@ KNativePtr Kotlin_AtomicNativePtr_get(KRef thiz) {
 }
 
 void Kotlin_AtomicReference_checkIfFrozen(KRef value) {
-    if (value != nullptr && !value->container()->permanentOrFrozen()) {
+    if (value != nullptr && !PermanentOrFrozen(value)) {
         ThrowInvalidMutabilityException(value);
     }
 }

runtime/src/main/cpp/Memory.cpp

@@ -180,10 +180,10 @@ public:
   }
 
   static int toIndex(const ContainerHeader* header) {
+    if (header == nullptr) return 2; // permanent.
     switch (header->tag()) {
       case CONTAINER_TAG_NORMAL   : return 0;
       case CONTAINER_TAG_STACK    : return 1;
-      case CONTAINER_TAG_PERMANENT: return 2;
       case CONTAINER_TAG_FROZEN:    return 3;
     }
     RuntimeAssert(false, "unknown container type");
@@ -381,15 +381,15 @@ THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr;
 constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**);
 
 inline bool isFreeable(const ContainerHeader* header) {
-  return header->tag() < CONTAINER_TAG_PERMANENT;
+  return header != nullptr && header->tag() != CONTAINER_TAG_STACK;
 }
 
 inline bool isArena(const ContainerHeader* header) {
-  return header->stack();
+  return header != nullptr && header->stack();
 }
 
 inline bool isAggregatingFrozenContainer(const ContainerHeader* header) {
-  return header->frozen() && header->objectCount() > 1;
+  return header != nullptr && header->frozen() && header->objectCount() > 1;
 }
 
 inline container_size_t alignUp(container_size_t size, int alignment) {
@@ -446,8 +446,8 @@ void KRefSharedHolder::verifyRefOwner() const {
     // Initialized runtime is required to throw the exception below
     // or to provide proper execution context for shared objects:
     if (memoryState == nullptr) Kotlin_initRuntimeIfNeeded();
-
-    if (!obj_->container()->shareable()) {
+    auto* container = obj_->container();
+    if (!Shareable(container)) {
       // TODO: add some info about the owner.
       ThrowIllegalObjectSharingException(obj_->type_info(), obj_);
     }
@@ -456,13 +456,6 @@ void KRefSharedHolder::verifyRefOwner() const {
 
 extern "C" {
 
-// Ensure LLVM never throws theStaticObjectsContainer away.
-// TODO: although practically const, marking it as such makes LLVM crazy, fix it.
-RUNTIME_USED ContainerHeader theStaticObjectsContainer = {
-  CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT,
-  0 /* Object count */
-};
-
 void objc_release(void* ptr);
 void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
 RUNTIME_NORETURN void ThrowFreezingException(KRef toFreeze, KRef blocker);
@@ -639,9 +632,9 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet*
              header->refCount_ >> CONTAINER_TAG_SHIFT)
   seen->insert(header);
   traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) {
-    auto child = ref->container();
+    auto* child = ref->container();
     RuntimeAssert(!isArena(child), "A reference to local object is encountered");
-    if (!child->permanent() && (seen->count(child) == 0)) {
+    if (child != nullptr && (seen->count(child) == 0)) {
       dumpWorker(prefix, child, seen);
     }
   });
@@ -650,9 +643,8 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet*
 void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
   ContainerHeaderSet seen;
   for (auto container : *roots) {
-    MEMORY_LOG("%p: %s%s%s\n", container,
+    MEMORY_LOG("%p: %s%s\n", container,
         container->frozen() ? "frozen " : "",
-        container->permanent() ? "permanent " : "",
         container->stack() ? "stack " : "")
     dumpWorker(prefix, container, &seen);
   }
@@ -696,8 +688,7 @@ void MarkGray(ContainerHeader* start) {
     traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
       auto childContainer = ref->container();
       RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
-
-      if (!childContainer->shareable()) {
+      if (!Shareable(childContainer)) {
         childContainer->decRefCount<false>();
         toVisit.push_front(childContainer);
       }
@@ -723,7 +714,7 @@ void ScanBlack(ContainerHeader* start) {
     traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
         auto childContainer = ref->container();
         RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
-        if (!childContainer->shareable()) {
+        if (!Shareable(childContainer)) {
           childContainer->incRefCount<false>();
           if (useColor) {
             int color = childContainer->color();
@@ -802,7 +793,7 @@ void Scan(ContainerHeader* start) {
      traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
        auto* childContainer = ref->container();
        RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
-       if (!childContainer->shareable()) {
+       if (!Shareable(childContainer)) {
          toVisit.push_front(childContainer);
        }
      });
@@ -823,7 +814,7 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) {
         if (ref == nullptr) return;
         auto* childContainer = ref->container();
         RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
-        if (childContainer->shareable()) {
+        if (Shareable(childContainer)) {
           UpdateRef(location, nullptr);
         } else {
           toVisit.push_front(childContainer);
@@ -840,7 +831,6 @@ inline void AddRef(ContainerHeader* header) {
   // (non-escaping stack objects, constant objects).
   switch (header->refCount_ & CONTAINER_TAG_MASK) {
     case CONTAINER_TAG_STACK:
-    case CONTAINER_TAG_PERMANENT:
       break;
     case CONTAINER_TAG_NORMAL:
       IncrementRC</* Atomic = */ false>(header);
@@ -856,7 +846,6 @@ inline void ReleaseRef(ContainerHeader* header) {
   // Looking at container type we may want to skip ReleaseRef() totally
   // (non-escaping stack objects, constant objects).
   switch (header->tag()) {
-    case CONTAINER_TAG_PERMANENT:
     case CONTAINER_TAG_STACK:
       break;
     case CONTAINER_TAG_NORMAL:
@@ -901,6 +890,7 @@ inline size_t containerSize(const ContainerHeader* container) {
 MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
   MetaObjHeader* meta = konanConstructInstance<MetaObjHeader>();
   TypeInfo* typeInfo = *location;
+  RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
   meta->typeInfo_ = typeInfo;
 #if KONAN_NO_THREADS
   *location = reinterpret_cast<TypeInfo*>(meta);
@@ -916,7 +906,7 @@ MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
 }
 
 void ObjHeader::destroyMetaObject(TypeInfo** location) {
-  MetaObjHeader* meta = *(reinterpret_cast<MetaObjHeader**>(location));
+  MetaObjHeader* meta = clearPointerBits(*(reinterpret_cast<MetaObjHeader**>(location)), OBJECT_TAG_MASK);
   *const_cast<const TypeInfo**>(location) = meta->typeInfo_;
   if (meta->counter_ != nullptr) {
     WeakReferenceCounterClear(meta->counter_);
@@ -947,13 +937,13 @@ ContainerHeader* AllocContainer(size_t size) {
 
 ContainerHeader* AllocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& containers) {
   auto componentSize = containers.size();
-  auto superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize);
-  auto place = reinterpret_cast<ContainerHeader**>(superContainer + 1);
+  auto* superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize);
+  auto* place = reinterpret_cast<ContainerHeader**>(superContainer + 1);
   for (auto* container : containers) {
     *place++ = container;
     // Set link to the new container.
-    auto obj = reinterpret_cast<ObjHeader*>(container + 1);
-    obj->container_ = superContainer;
+    auto* obj = reinterpret_cast<ObjHeader*>(container + 1);
+    obj->setContainer(superContainer);
     MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer);
   }
   superContainer->setObjectCount(componentSize);
@@ -962,7 +952,7 @@ ContainerHeader* AllocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& c
 }
 
 void FreeAggregatingFrozenContainer(ContainerHeader* container) {
-  auto state = memoryState;
+  auto* state = memoryState;
   RuntimeAssert(isAggregatingFrozenContainer(container), "expected fictitious frozen container");
   MEMORY_LOG("%p is fictitious frozen container\n", container);
   RuntimeAssert(!container->buffered(), "frozen objects must not participate in GC")
@@ -985,7 +975,7 @@ void FreeAggregatingFrozenContainer(ContainerHeader* container) {
 }
 
 void FreeContainer(ContainerHeader* container) {
-  RuntimeAssert(!container->permanent(), "this kind of container shalln't be freed");
+  RuntimeAssert(container != nullptr, "this kind of container shalln't be freed");
   auto state = memoryState;
 
   CONTAINER_FREE_EVENT(state, container)
@@ -1138,13 +1128,19 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun
 }
 
 inline void AddRef(const ObjHeader* object) {
-  MEMORY_LOG("AddRef on %p in %p\n", object, object->container())
-  AddRef(object->container());
+  auto* container = object->container();
+  if (container != nullptr) {
+    MEMORY_LOG("AddRef on %p in %p\n", object, container)
+    AddRef(container);
+  }
 }
 
 inline void ReleaseRef(const ObjHeader* object) {
-  MEMORY_LOG("ReleaseRef on %p in %p\n", object, object->container())
-  ReleaseRef(object->container());
+  auto* container = object->container();
+  if (container != nullptr) {
+    MEMORY_LOG("ReleaseRef on %p in %p\n", object, container)
+    ReleaseRef(container);
+  }
 }
 
 void AddRefFromAssociatedObject(const ObjHeader* object) {
@@ -1162,10 +1158,6 @@ MemoryState* InitMemory() {
                 ==
                 offsetof(ObjHeader,   typeInfoOrMeta_),
                 "Layout mismatch");
-  RuntimeAssert(offsetof(ArrayHeader, container_)
-                ==
-                offsetof(ObjHeader  , container_),
-                "Layout mismatch");
   RuntimeAssert(offsetof(TypeInfo, typeInfo_)
                 ==
                 offsetof(MetaObjHeader, typeInfo_),
@@ -1357,7 +1349,7 @@ ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot)
 ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) {
   if (param == nullptr) return localSlot;
   auto container = param->container();
-  if ((container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK)
+  if (container == nullptr || (container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK)
     return localSlot;
   auto chunk = reinterpret_cast<ContainerChunk*>(container) - 1;
   return reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(&chunk->arena) | ARENA_BIT);
@@ -1556,7 +1548,7 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) {
 
 KNativePtr CreateStablePointer(KRef any) {
   if (any == nullptr) return nullptr;
-  AddRef(any->container());
+  AddRef(any);
   return reinterpret_cast<KNativePtr>(any);
 }
 
@@ -1593,7 +1585,7 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) {
     if (container->refCount() != 0) return true;
     traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) {
         auto* child = ref->container();
-        if (!child->shareable() && (visited->count(child) == 0)) {
+        if (!Shareable(child) && (visited->count(child) == 0)) {
            toVisit.push_front(child);
         }
      });
@@ -1608,7 +1600,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
     auto state = memoryState;
     auto* container = root->container();
 
-    if (container->frozen())
+    if (container == nullptr || container->frozen())
       // We assume, that frozen objects can be safely passed and are already removed
       // GC candidate list.
       return true;
@@ -1617,7 +1609,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
     if (!checked) {
       hasExternalRefs(container, &visited);
     } else {
-      if (!container->shareable()) {
+      if (!Shareable(container)) {
         container->decRefCount<false>();
         MarkGray<false>(container);
         auto bad = hasExternalRefs(container, &visited);
@@ -1661,7 +1653,7 @@ void depthFirstTraversal(ContainerHeader* container, bool* hasCycles,
           return;
       }
       ContainerHeader* objContainer = obj->container();
-      if (!objContainer->shareable()) {
+      if (!Shareable(objContainer)) {
         // Marked GREY, there's cycle.
         if (objContainer->seen()) *hasCycles = true;
 
@@ -1704,7 +1696,7 @@ void freezeAcyclic(ContainerHeader* rootContainer) {
     current->freeze();
     traverseContainerReferredObjects(current, [current, &queue](ObjHeader* obj) {
         ContainerHeader* objContainer = obj->container();
-        if (!objContainer->shareable()) {
+        if (!Shareable(objContainer)) {
           if (objContainer->marked())
             queue.push_back(objContainer);
         }
@@ -1723,7 +1715,7 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
     reversedEdges.emplace(current, KStdVector<ContainerHeader*>(0));
     traverseContainerReferredObjects(current, [current, &queue, &reversedEdges](ObjHeader* obj) {
           ContainerHeader* objContainer = obj->container();
-          if (!objContainer->shareable()) {
+          if (!Shareable(objContainer)) {
             if (objContainer->marked())
               queue.push_back(objContainer);
             reversedEdges.emplace(objContainer, KStdVector<ContainerHeader*>(0)).first->second.push_back(current);
@@ -1755,14 +1747,11 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
     for (auto* container : component) {
       totalCount += container->refCount();
       traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) {
-          if (!obj->container()->shareable())
+          auto* container = obj->container();
+          if (!Shareable(container))
               ++internalRefsCount;
         });
       }
-    // Create fictitious container for the whole component.
-    auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component);
-    // Don't count internal references.
-    superContainer->setRefCount(totalCount - internalRefsCount);
 
     // Freeze component.
     for (auto* container : component) {
@@ -1771,7 +1760,14 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
       // Note, that once object is frozen, it could be concurrently accessed, so
       // color and similar attributes shall not be used.
       container->freeze();
+      // We set refcount of original container to zero, so that it is seen as such after removal
+      // meta-object, where aggregating container is stored.
+      container->setRefCount(0);
     }
+    // Create fictitious container for the whole component.
+    auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component);
+    // Don't count internal references.
+    superContainer->setRefCount(totalCount - internalRefsCount);
   }
 }
 
@@ -1803,7 +1799,7 @@ void FreezeSubgraph(ObjHeader* root) {
   // First check that passed object graph has no cycles.
   // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir.
   ContainerHeader* rootContainer = root->container();
-  if (rootContainer->shareable()) return;
+  if (Shareable(rootContainer)) return;
 
   // Do DFS cycle detection.
   bool hasCycles = false;
@@ -1836,7 +1832,8 @@ void FreezeSubgraph(ObjHeader* root) {
 // This function is called from field mutators to check if object's header is frozen.
 // If object is frozen, an exception is thrown.
 void MutationCheck(ObjHeader* obj) {
-  if (obj->container()->frozen()) ThrowInvalidMutabilityException(obj);
+  auto* container = obj->container();
+  if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj);
 }
 
 OBJ_GETTER(SwapRefLocked,
@@ -1881,7 +1878,8 @@ OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) {
 }
 
 void EnsureNeverFrozen(ObjHeader* object) {
-   if (object->container()->frozen())
+   auto* container = object->container();
+   if (container == nullptr || container->frozen())
       ThrowFreezingException(object, object);
    // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency,
    // although there's subtle race with case, where other thread freezes the same object after check.
@@ -1889,15 +1887,16 @@ void EnsureNeverFrozen(ObjHeader* object) {
 }
 
 KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) {
-    RuntimeAssert(where->container()->frozen(), "Must be used on frozen objects only");
-    RuntimeAssert(what == nullptr || what->container()->permanentOrFrozen(),
+    RuntimeAssert(where->container() != nullptr && where->container()->frozen(), "Must be used on frozen objects only");
+    RuntimeAssert(what == nullptr || PermanentOrFrozen(what),
         "Must be used with an immutable value");
     if (what != nullptr) {
         // Now we check that `where` is not reachable from `what`.
         // As we cannot modify objects while traversing, instead we remember all seen objects in a set.
         KStdUnorderedSet<ContainerHeader*> seen;
         KStdDeque<ContainerHeader*> queue;
-        queue.push_back(what->container());
+        if (what->container() != nullptr)
+            queue.push_back(what->container());
         bool acyclic = true;
         while (!queue.empty() && acyclic) {
             ContainerHeader* current = queue.front();
@@ -1915,7 +1914,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what
                     acyclic = false;
                 } else {
                     auto* objContainer = obj->container();
-                    if (seen.count(objContainer) == 0)
+                    if (objContainer != nullptr && seen.count(objContainer) == 0)
                         queue.push_back(objContainer);
                 }
               });
@@ -1930,8 +1929,8 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what
 }
 
 void Kotlin_Any_share(ObjHeader* obj) {
-    auto container = obj->container();
-    if (container->shareable()) return;
+    auto* container = obj->container();
+    if (Shareable(container)) return;
     RuntimeCheck(container->objectCount() == 1, "Must be a single object container");
     container->makeShareable();
 }

runtime/src/main/cpp/Memory.h

@@ -31,19 +31,14 @@ typedef enum {
   // Stack container, no need to free, children cleanup still shall be there.
   CONTAINER_TAG_STACK = 2,
   // Atomic container, reference counter is atomically updated.
-  CONTAINER_TAG_ATOMIC = 5 | 1,  // shareable
-  // Those container tags shall not be refcounted.
-  // Permanent container, cannot refer to non-permanent containers, so no need to cleanup those.
-  // Please check isFreeable() if changing the numeric value.
-  CONTAINER_TAG_PERMANENT = 7 | 1,  // shareable
+  CONTAINER_TAG_ATOMIC = 3 | 1,  // shareable
   // Shift to get actual counter.
-  CONTAINER_TAG_SHIFT = 3,
+  CONTAINER_TAG_SHIFT = 2,
   // Actual value to increment/decrement container by. Tag is in lower bits.
   CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
   // Mask for container type.
   CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
 
-  // Those bit masks are applied to objectCount_ field.
   // Shift to get actual object count.
   CONTAINER_TAG_GC_SHIFT     = 6,
   CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT,
@@ -72,6 +67,14 @@ typedef enum {
   CONTAINER_TAG_GC_SEEN     = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2)
 } ContainerTag;
 
+typedef enum {
+  // Must match to permTag() in Kotlin.
+  OBJECT_TAG_PERMANENT_CONTAINER = 1 << 0,
+  OBJECT_TAG_NONTRIVIAL_CONTAINER = 1 << 1,
+  // Keep in sync with immTypeInfoMask in Kotlin.
+  OBJECT_TAG_MASK = (1 << 2) - 1
+} ObjectTag;
+
 typedef uint32_t container_size_t;
 
 // Header of all container objects. Contains reference counter.
@@ -86,10 +89,6 @@ struct ContainerHeader {
       return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NORMAL;
   }
 
-  inline bool permanent() const {
-    return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
-  }
-
   inline bool frozen() const {
     return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN;
   }
@@ -102,12 +101,8 @@ struct ContainerHeader {
       refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_ATOMIC;
   }
 
-  inline bool permanentOrFrozen() const {
-    return tag() == CONTAINER_TAG_PERMANENT || tag() == CONTAINER_TAG_FROZEN;
-  }
-
   inline bool shareable() const {
-      return (tag() & 1) != 0; // CONTAINER_TAG_PERMANENT || CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC
+      return (tag() & 1) != 0; // CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC
   }
 
   inline bool stack() const {
@@ -228,29 +223,83 @@ struct ContainerHeader {
   }
 };
 
+inline bool PermanentOrFrozen(ContainerHeader* container) {
+    return container == nullptr || container->frozen();
+}
+
+inline bool Shareable(ContainerHeader* container) {
+    return container == nullptr || container->shareable();
+}
+
 struct ArrayHeader;
 struct MetaObjHeader;
 
+template <typename T>
+ALWAYS_INLINE T* setPointerBits(T* ptr, unsigned bits) {
+  return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) | bits);
+}
+
+template <typename T>
+ALWAYS_INLINE T* clearPointerBits(T* ptr, unsigned bits) {
+  return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) & ~static_cast<uintptr_t>(bits));
+}
+
+template <typename T>
+ALWAYS_INLINE unsigned getPointerBits(T* ptr, unsigned bits) {
+  return reinterpret_cast<uintptr_t>(ptr) & static_cast<uintptr_t>(bits);
+}
+
+template <typename T>
+ALWAYS_INLINE bool hasPointerBits(T* ptr, unsigned bits) {
+  return getPointerBits(ptr, bits) != 0;
+}
+
+// Header for the meta-object.
+struct MetaObjHeader {
+  // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
+  const TypeInfo* typeInfo_;
+  // Strong reference to the counter object.
+  ObjHeader* counter_;
+  // Container pointer.
+  ContainerHeader* container_;
+#ifdef KONAN_OBJC_INTEROP
+  void* associatedObject_;
+#endif
+
+  // Flags for the object state.
+  int32_t flags_;
+};
+
 // Header of every object.
 struct ObjHeader {
   TypeInfo* typeInfoOrMeta_;
-  ContainerHeader* container_;
 
   const TypeInfo* type_info() const {
-    return typeInfoOrMeta_->typeInfo_;
+    return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
   }
 
   bool has_meta_object() const {
-    return typeInfoOrMeta_ != typeInfoOrMeta_->typeInfo_;
+    auto* typeInfoOrMeta = clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
+    return (typeInfoOrMeta != typeInfoOrMeta->typeInfo_);
   }
 
   MetaObjHeader* meta_object() {
      return has_meta_object() ?
-        reinterpret_cast<MetaObjHeader*>(typeInfoOrMeta_) : createMetaObject(&typeInfoOrMeta_);
+        reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)) :
+        createMetaObject(&typeInfoOrMeta_);
+  }
+
+  void setContainer(ContainerHeader* container) {
+    meta_object()->container_ = container;
+    typeInfoOrMeta_ = setPointerBits(typeInfoOrMeta_, OBJECT_TAG_NONTRIVIAL_CONTAINER);
   }
 
   ContainerHeader* container() const {
-    return container_;
+    unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
+    if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) return nullptr;
+    return (bits & OBJECT_TAG_NONTRIVIAL_CONTAINER) != 0 ?
+         (reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_ :
+         reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(this)) - 1;
   }
 
   // Unsafe cast to ArrayHeader. Use carefully!
@@ -258,7 +307,7 @@ struct ObjHeader {
   const ArrayHeader* array() const { return reinterpret_cast<const ArrayHeader*>(this); }
 
   inline bool permanent() const {
-    return container()->permanent();
+    return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
   }
 
   static MetaObjHeader* createMetaObject(TypeInfo** location);
@@ -268,14 +317,9 @@ struct ObjHeader {
 // Header of value type array objects. Keep layout in sync with that of object header.
 struct ArrayHeader {
   TypeInfo* typeInfoOrMeta_;
-  ContainerHeader* container_;
 
   const TypeInfo* type_info() const {
-    return typeInfoOrMeta_->typeInfo_;
-  }
-
-  ContainerHeader* container() const {
-    return container_;
+    return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
   }
 
   ObjHeader* obj() { return reinterpret_cast<ObjHeader*>(this); }
@@ -285,19 +329,10 @@ struct ArrayHeader {
   uint32_t count_;
 };
 
-// Header for the meta-object.
-struct MetaObjHeader {
-  // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
-  const TypeInfo* typeInfo_;
-  // Strong reference to counter object.
-  ObjHeader* counter_;
-  // Flags for object state.
-  int32_t flags_;
-
-#ifdef KONAN_OBJC_INTEROP
-  void* associatedObject_;
-#endif
-};
+inline bool PermanentOrFrozen(ObjHeader* obj) {
+    auto* container = obj->container();
+    return container == nullptr || container->frozen();
+}
 
 inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
   // Instance size is negative.
@@ -311,7 +346,6 @@ class Container {
   ContainerHeader* header_;
 
   void SetHeader(ObjHeader* obj, const TypeInfo* type_info) {
-    obj->container_ = header_;
     obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(type_info);
     // Take into account typeInfo's immutability for ARC strategy.
     if ((type_info->flags_ & TF_IMMUTABLE) != 0)
@@ -393,8 +427,8 @@ class ArenaContainer {
   bool allocContainer(container_size_t minSize);
 
   void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) {
-    obj->container_ = currentChunk_->asHeader();
     obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
+    obj->setContainer(currentChunk_->asHeader());
     // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC.
   }
 

runtime/src/main/cpp/ObjCExport.mm

@@ -113,7 +113,7 @@ static void initializeClass(Class clazz);
 extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
 
 static inline id AtomicSetAssociatedObject(ObjHeader* obj, id associatedObject) {
-  if (!obj->container()->permanentOrFrozen()) {
+  if (!PermanentOrFrozen(obj)) {
     SetAssociatedObject(obj, associatedObject);
     return associatedObject;
   } else {

runtime/src/main/cpp/Worker.cpp

@@ -547,7 +547,7 @@ void Kotlin_Worker_freezeInternal(KRef object) {
 }
 
 KBoolean Kotlin_Worker_isFrozenInternal(KRef object) {
-  return object == nullptr || object->container()->permanentOrFrozen();
+  return object == nullptr || PermanentOrFrozen(object);
 }
 
 void Kotlin_Worker_ensureNeverFrozen(KRef object) {

shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt

@@ -6,7 +6,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion {
 
 data class KonanAbiVersion(val version: Int) {
     companion object {
-        val CURRENT = KonanAbiVersion(4)
+        val CURRENT = KonanAbiVersion(5)
     }
     override fun toString() = "$version"
 }