From 9e3a68f8355815acffdd280bed849a9f337e58f0 Mon Sep 17 00:00:00 2001 From: Nikolay Igotti Date: Mon, 3 Dec 2018 13:50:56 +0300 Subject: [PATCH] Reduce size of object header by one pointer. (#2432) --- .../backend/konan/llvm/CodeGenerator.kt | 11 +- .../kotlin/backend/konan/llvm/ContextUtils.kt | 2 - .../backend/konan/llvm/StaticObjects.kt | 21 +-- runtime/src/main/cpp/Atomic.cpp | 2 +- runtime/src/main/cpp/Memory.cpp | 121 +++++++++--------- runtime/src/main/cpp/Memory.h | 120 ++++++++++------- runtime/src/main/cpp/ObjCExport.mm | 2 +- runtime/src/main/cpp/Worker.cpp | 2 +- .../jetbrains/kotlin/konan/KonanAbiVersion.kt | 2 +- 9 files changed, 162 insertions(+), 121 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt index d33101283b7..f3926a338db 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt @@ -9,7 +9,6 @@ package org.jetbrains.kotlin.backend.konan.llvm import kotlinx.cinterop.* import llvm.* import org.jetbrains.kotlin.backend.konan.* -import org.jetbrains.kotlin.backend.konan.descriptors.* import org.jetbrains.kotlin.backend.konan.descriptors.isInterface import org.jetbrains.kotlin.backend.konan.irasdescriptors.* import org.jetbrains.kotlin.backend.konan.irasdescriptors.ClassConstructorDescriptor @@ -28,6 +27,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils { fun llvmFunction(function: FunctionDescriptor): LLVMValueRef = function.llvmFunction val intPtrType = LLVMIntPtrType(llvmTargetData)!! internal val immOneIntPtrType = LLVMConstInt(intPtrType, 1, 1)!! + // Keep in sync with OBJECT_TAG_MASK in C++. + internal val immTypeInfoMask = LLVMConstNot(LLVMConstInt(intPtrType, 3, 0)!!)!! //-------------------------------------------------------------------------// @@ -624,7 +625,11 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, call(context.llvm.getObjCKotlinTypeInfo, listOf(receiver)) } else { val typeInfoOrMetaPtr = structGep(receiver, 0 /* typeInfoOrMeta_ */) - val typeInfoOrMeta = load(typeInfoOrMetaPtr) + val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr) + // Clear two lower bits. + val typeInfoOrMetaWithFlagsRaw = ptrToInt(typeInfoOrMetaWithFlags, codegen.intPtrType) + val typeInfoOrMetaRaw = and(typeInfoOrMetaWithFlagsRaw, codegen.immTypeInfoMask) + val typeInfoOrMeta = intToPtr(typeInfoOrMetaRaw, kTypeInfoPtr) val typeInfoPtrPtr = structGep(typeInfoOrMeta, 0 /* typeInfo */) load(typeInfoPtrPtr) } @@ -634,7 +639,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef, /* * Resolve owner of the call with special handling of Any methods: * if toString/eq/hc is invoked on an interface instance, we resolve - * owner as Any and dispatch it via vtable + * owner as Any and dispatch it via vtable. */ val anyMethod = (descriptor as SimpleFunctionDescriptor).findOverriddenMethodOfAny() val owner = (anyMethod ?: descriptor).containingDeclaration as ClassDescriptor diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt index 574b9805380..1f65dff2347 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt @@ -457,8 +457,6 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) { origin = context.standardLlvmSymbolsOrigin ) - val staticContainer = importRtGlobal("theStaticObjectsContainer") - val memsetFunction = importMemset() val usedFunctions = mutableListOf() diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt index 23e83b4d1dc..3fdb107e194 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt @@ -5,10 +5,8 @@ package org.jetbrains.kotlin.backend.konan.llvm -import llvm.LLVMLinkage -import llvm.LLVMSetLinkage -import llvm.LLVMTypeRef -import llvm.LLVMValueRef +import kotlinx.cinterop.cValuesOf +import llvm.* import org.jetbrains.kotlin.backend.konan.irasdescriptors.fqNameSafe import org.jetbrains.kotlin.backend.konan.irasdescriptors.llvmSymbolOrigin import org.jetbrains.kotlin.descriptors.ClassDescriptor @@ -17,16 +15,23 @@ import org.jetbrains.kotlin.ir.declarations.IrClass import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.Name +private fun ConstPointer.add(index: Int): ConstPointer { + return constPointer(LLVMConstGEP(llvm, cValuesOf(Int32(index).llvm), 1)!!) +} + +// Must match OBJECT_TAG_PERMANENT_CONTAINER in C++. +private fun StaticData.permanentTag(typeInfo: ConstPointer): ConstPointer { + // Only pointer arithmetic via GEP works on constant pointers in LLVM. + return typeInfo.bitcast(int8TypePtr).add(1).bitcast(kTypeInfoPtr) +} private fun StaticData.objHeader(typeInfo: ConstPointer): Struct { - val container = constValue(context.llvm.staticContainer) - return Struct(runtime.objHeaderType, typeInfo, container) + return Struct(runtime.objHeaderType, permanentTag(typeInfo)) } private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct { assert (length >= 0) - val container = constValue(context.llvm.staticContainer) - return Struct(runtime.arrayHeaderType, typeInfo, container, Int32(length)) + return Struct(runtime.arrayHeaderType, permanentTag(typeInfo), Int32(length)) } internal fun StaticData.createKotlinStringLiteral(value: String): ConstPointer { diff --git a/runtime/src/main/cpp/Atomic.cpp b/runtime/src/main/cpp/Atomic.cpp index 41dee6ec815..c10f342b913 100644 --- a/runtime/src/main/cpp/Atomic.cpp +++ b/runtime/src/main/cpp/Atomic.cpp @@ -168,7 +168,7 @@ KNativePtr Kotlin_AtomicNativePtr_get(KRef thiz) { } void Kotlin_AtomicReference_checkIfFrozen(KRef value) { - if (value != nullptr && !value->container()->permanentOrFrozen()) { + if (value != nullptr && !PermanentOrFrozen(value)) { ThrowInvalidMutabilityException(value); } } diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 40db5748089..c6091aecb3b 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -180,10 +180,10 @@ public: } static int toIndex(const ContainerHeader* header) { + if (header == nullptr) return 2; // permanent. switch (header->tag()) { case CONTAINER_TAG_NORMAL : return 0; case CONTAINER_TAG_STACK : return 1; - case CONTAINER_TAG_PERMANENT: return 2; case CONTAINER_TAG_FROZEN: return 3; } RuntimeAssert(false, "unknown container type"); @@ -381,15 +381,15 @@ THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr; constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**); inline bool isFreeable(const ContainerHeader* header) { - return header->tag() < CONTAINER_TAG_PERMANENT; + return header != nullptr && header->tag() != CONTAINER_TAG_STACK; } inline bool isArena(const ContainerHeader* header) { - return header->stack(); + return header != nullptr && header->stack(); } inline bool isAggregatingFrozenContainer(const ContainerHeader* header) { - return header->frozen() && header->objectCount() > 1; + return header != nullptr && header->frozen() && header->objectCount() > 1; } inline container_size_t alignUp(container_size_t size, int alignment) { @@ -446,8 +446,8 @@ void KRefSharedHolder::verifyRefOwner() const { // Initialized runtime is required to throw the exception below // or to provide proper execution context for shared objects: if (memoryState == nullptr) Kotlin_initRuntimeIfNeeded(); - - if (!obj_->container()->shareable()) { + auto* container = obj_->container(); + if (!Shareable(container)) { // TODO: add some info about the owner. ThrowIllegalObjectSharingException(obj_->type_info(), obj_); } @@ -456,13 +456,6 @@ void KRefSharedHolder::verifyRefOwner() const { extern "C" { -// Ensure LLVM never throws theStaticObjectsContainer away. -// TODO: although practically const, marking it as such makes LLVM crazy, fix it. -RUNTIME_USED ContainerHeader theStaticObjectsContainer = { - CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT, - 0 /* Object count */ -}; - void objc_release(void* ptr); void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject); RUNTIME_NORETURN void ThrowFreezingException(KRef toFreeze, KRef blocker); @@ -639,9 +632,9 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* header->refCount_ >> CONTAINER_TAG_SHIFT) seen->insert(header); traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) { - auto child = ref->container(); + auto* child = ref->container(); RuntimeAssert(!isArena(child), "A reference to local object is encountered"); - if (!child->permanent() && (seen->count(child) == 0)) { + if (child != nullptr && (seen->count(child) == 0)) { dumpWorker(prefix, child, seen); } }); @@ -650,9 +643,8 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { ContainerHeaderSet seen; for (auto container : *roots) { - MEMORY_LOG("%p: %s%s%s\n", container, + MEMORY_LOG("%p: %s%s\n", container, container->frozen() ? "frozen " : "", - container->permanent() ? "permanent " : "", container->stack() ? "stack " : "") dumpWorker(prefix, container, &seen); } @@ -696,8 +688,7 @@ void MarkGray(ContainerHeader* start) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { auto childContainer = ref->container(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); - - if (!childContainer->shareable()) { + if (!Shareable(childContainer)) { childContainer->decRefCount(); toVisit.push_front(childContainer); } @@ -723,7 +714,7 @@ void ScanBlack(ContainerHeader* start) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { auto childContainer = ref->container(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); - if (!childContainer->shareable()) { + if (!Shareable(childContainer)) { childContainer->incRefCount(); if (useColor) { int color = childContainer->color(); @@ -802,7 +793,7 @@ void Scan(ContainerHeader* start) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { auto* childContainer = ref->container(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); - if (!childContainer->shareable()) { + if (!Shareable(childContainer)) { toVisit.push_front(childContainer); } }); @@ -823,7 +814,7 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) { if (ref == nullptr) return; auto* childContainer = ref->container(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); - if (childContainer->shareable()) { + if (Shareable(childContainer)) { UpdateRef(location, nullptr); } else { toVisit.push_front(childContainer); @@ -840,7 +831,6 @@ inline void AddRef(ContainerHeader* header) { // (non-escaping stack objects, constant objects). switch (header->refCount_ & CONTAINER_TAG_MASK) { case CONTAINER_TAG_STACK: - case CONTAINER_TAG_PERMANENT: break; case CONTAINER_TAG_NORMAL: IncrementRC(header); @@ -856,7 +846,6 @@ inline void ReleaseRef(ContainerHeader* header) { // Looking at container type we may want to skip ReleaseRef() totally // (non-escaping stack objects, constant objects). switch (header->tag()) { - case CONTAINER_TAG_PERMANENT: case CONTAINER_TAG_STACK: break; case CONTAINER_TAG_NORMAL: @@ -901,6 +890,7 @@ inline size_t containerSize(const ContainerHeader* container) { MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { MetaObjHeader* meta = konanConstructInstance(); TypeInfo* typeInfo = *location; + RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged"); meta->typeInfo_ = typeInfo; #if KONAN_NO_THREADS *location = reinterpret_cast(meta); @@ -916,7 +906,7 @@ MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { } void ObjHeader::destroyMetaObject(TypeInfo** location) { - MetaObjHeader* meta = *(reinterpret_cast(location)); + MetaObjHeader* meta = clearPointerBits(*(reinterpret_cast(location)), OBJECT_TAG_MASK); *const_cast(location) = meta->typeInfo_; if (meta->counter_ != nullptr) { WeakReferenceCounterClear(meta->counter_); @@ -947,13 +937,13 @@ ContainerHeader* AllocContainer(size_t size) { ContainerHeader* AllocAggregatingFrozenContainer(KStdVector& containers) { auto componentSize = containers.size(); - auto superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize); - auto place = reinterpret_cast(superContainer + 1); + auto* superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize); + auto* place = reinterpret_cast(superContainer + 1); for (auto* container : containers) { *place++ = container; // Set link to the new container. - auto obj = reinterpret_cast(container + 1); - obj->container_ = superContainer; + auto* obj = reinterpret_cast(container + 1); + obj->setContainer(superContainer); MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer); } superContainer->setObjectCount(componentSize); @@ -962,7 +952,7 @@ ContainerHeader* AllocAggregatingFrozenContainer(KStdVector& c } void FreeAggregatingFrozenContainer(ContainerHeader* container) { - auto state = memoryState; + auto* state = memoryState; RuntimeAssert(isAggregatingFrozenContainer(container), "expected fictitious frozen container"); MEMORY_LOG("%p is fictitious frozen container\n", container); RuntimeAssert(!container->buffered(), "frozen objects must not participate in GC") @@ -985,7 +975,7 @@ void FreeAggregatingFrozenContainer(ContainerHeader* container) { } void FreeContainer(ContainerHeader* container) { - RuntimeAssert(!container->permanent(), "this kind of container shalln't be freed"); + RuntimeAssert(container != nullptr, "this kind of container shalln't be freed"); auto state = memoryState; CONTAINER_FREE_EVENT(state, container) @@ -1138,13 +1128,19 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun } inline void AddRef(const ObjHeader* object) { - MEMORY_LOG("AddRef on %p in %p\n", object, object->container()) - AddRef(object->container()); + auto* container = object->container(); + if (container != nullptr) { + MEMORY_LOG("AddRef on %p in %p\n", object, container) + AddRef(container); + } } inline void ReleaseRef(const ObjHeader* object) { - MEMORY_LOG("ReleaseRef on %p in %p\n", object, object->container()) - ReleaseRef(object->container()); + auto* container = object->container(); + if (container != nullptr) { + MEMORY_LOG("ReleaseRef on %p in %p\n", object, container) + ReleaseRef(container); + } } void AddRefFromAssociatedObject(const ObjHeader* object) { @@ -1162,10 +1158,6 @@ MemoryState* InitMemory() { == offsetof(ObjHeader, typeInfoOrMeta_), "Layout mismatch"); - RuntimeAssert(offsetof(ArrayHeader, container_) - == - offsetof(ObjHeader , container_), - "Layout mismatch"); RuntimeAssert(offsetof(TypeInfo, typeInfo_) == offsetof(MetaObjHeader, typeInfo_), @@ -1357,7 +1349,7 @@ ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) { if (param == nullptr) return localSlot; auto container = param->container(); - if ((container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK) + if (container == nullptr || (container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK) return localSlot; auto chunk = reinterpret_cast(container) - 1; return reinterpret_cast(reinterpret_cast(&chunk->arena) | ARENA_BIT); @@ -1556,7 +1548,7 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) { KNativePtr CreateStablePointer(KRef any) { if (any == nullptr) return nullptr; - AddRef(any->container()); + AddRef(any); return reinterpret_cast(any); } @@ -1593,7 +1585,7 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) { if (container->refCount() != 0) return true; traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) { auto* child = ref->container(); - if (!child->shareable() && (visited->count(child) == 0)) { + if (!Shareable(child) && (visited->count(child) == 0)) { toVisit.push_front(child); } }); @@ -1608,7 +1600,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) { auto state = memoryState; auto* container = root->container(); - if (container->frozen()) + if (container == nullptr || container->frozen()) // We assume, that frozen objects can be safely passed and are already removed // GC candidate list. return true; @@ -1617,7 +1609,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) { if (!checked) { hasExternalRefs(container, &visited); } else { - if (!container->shareable()) { + if (!Shareable(container)) { container->decRefCount(); MarkGray(container); auto bad = hasExternalRefs(container, &visited); @@ -1661,7 +1653,7 @@ void depthFirstTraversal(ContainerHeader* container, bool* hasCycles, return; } ContainerHeader* objContainer = obj->container(); - if (!objContainer->shareable()) { + if (!Shareable(objContainer)) { // Marked GREY, there's cycle. if (objContainer->seen()) *hasCycles = true; @@ -1704,7 +1696,7 @@ void freezeAcyclic(ContainerHeader* rootContainer) { current->freeze(); traverseContainerReferredObjects(current, [current, &queue](ObjHeader* obj) { ContainerHeader* objContainer = obj->container(); - if (!objContainer->shareable()) { + if (!Shareable(objContainer)) { if (objContainer->marked()) queue.push_back(objContainer); } @@ -1723,7 +1715,7 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector(0)); traverseContainerReferredObjects(current, [current, &queue, &reversedEdges](ObjHeader* obj) { ContainerHeader* objContainer = obj->container(); - if (!objContainer->shareable()) { + if (!Shareable(objContainer)) { if (objContainer->marked()) queue.push_back(objContainer); reversedEdges.emplace(objContainer, KStdVector(0)).first->second.push_back(current); @@ -1755,14 +1747,11 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVectorrefCount(); traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) { - if (!obj->container()->shareable()) + auto* container = obj->container(); + if (!Shareable(container)) ++internalRefsCount; }); } - // Create fictitious container for the whole component. - auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component); - // Don't count internal references. - superContainer->setRefCount(totalCount - internalRefsCount); // Freeze component. for (auto* container : component) { @@ -1771,7 +1760,14 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVectorfreeze(); + // We set refcount of original container to zero, so that it is seen as such after removal + // meta-object, where aggregating container is stored. + container->setRefCount(0); } + // Create fictitious container for the whole component. + auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component); + // Don't count internal references. + superContainer->setRefCount(totalCount - internalRefsCount); } } @@ -1803,7 +1799,7 @@ void FreezeSubgraph(ObjHeader* root) { // First check that passed object graph has no cycles. // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir. ContainerHeader* rootContainer = root->container(); - if (rootContainer->shareable()) return; + if (Shareable(rootContainer)) return; // Do DFS cycle detection. bool hasCycles = false; @@ -1836,7 +1832,8 @@ void FreezeSubgraph(ObjHeader* root) { // This function is called from field mutators to check if object's header is frozen. // If object is frozen, an exception is thrown. void MutationCheck(ObjHeader* obj) { - if (obj->container()->frozen()) ThrowInvalidMutabilityException(obj); + auto* container = obj->container(); + if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj); } OBJ_GETTER(SwapRefLocked, @@ -1881,7 +1878,8 @@ OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) { } void EnsureNeverFrozen(ObjHeader* object) { - if (object->container()->frozen()) + auto* container = object->container(); + if (container == nullptr || container->frozen()) ThrowFreezingException(object, object); // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency, // although there's subtle race with case, where other thread freezes the same object after check. @@ -1889,15 +1887,16 @@ void EnsureNeverFrozen(ObjHeader* object) { } KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) { - RuntimeAssert(where->container()->frozen(), "Must be used on frozen objects only"); - RuntimeAssert(what == nullptr || what->container()->permanentOrFrozen(), + RuntimeAssert(where->container() != nullptr && where->container()->frozen(), "Must be used on frozen objects only"); + RuntimeAssert(what == nullptr || PermanentOrFrozen(what), "Must be used with an immutable value"); if (what != nullptr) { // Now we check that `where` is not reachable from `what`. // As we cannot modify objects while traversing, instead we remember all seen objects in a set. KStdUnorderedSet seen; KStdDeque queue; - queue.push_back(what->container()); + if (what->container() != nullptr) + queue.push_back(what->container()); bool acyclic = true; while (!queue.empty() && acyclic) { ContainerHeader* current = queue.front(); @@ -1915,7 +1914,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what acyclic = false; } else { auto* objContainer = obj->container(); - if (seen.count(objContainer) == 0) + if (objContainer != nullptr && seen.count(objContainer) == 0) queue.push_back(objContainer); } }); @@ -1930,8 +1929,8 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what } void Kotlin_Any_share(ObjHeader* obj) { - auto container = obj->container(); - if (container->shareable()) return; + auto* container = obj->container(); + if (Shareable(container)) return; RuntimeCheck(container->objectCount() == 1, "Must be a single object container"); container->makeShareable(); } diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 806ca887f40..0163ff00d67 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -31,19 +31,14 @@ typedef enum { // Stack container, no need to free, children cleanup still shall be there. CONTAINER_TAG_STACK = 2, // Atomic container, reference counter is atomically updated. - CONTAINER_TAG_ATOMIC = 5 | 1, // shareable - // Those container tags shall not be refcounted. - // Permanent container, cannot refer to non-permanent containers, so no need to cleanup those. - // Please check isFreeable() if changing the numeric value. - CONTAINER_TAG_PERMANENT = 7 | 1, // shareable + CONTAINER_TAG_ATOMIC = 3 | 1, // shareable // Shift to get actual counter. - CONTAINER_TAG_SHIFT = 3, + CONTAINER_TAG_SHIFT = 2, // Actual value to increment/decrement container by. Tag is in lower bits. CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT, // Mask for container type. CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1, - // Those bit masks are applied to objectCount_ field. // Shift to get actual object count. CONTAINER_TAG_GC_SHIFT = 6, CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT, @@ -72,6 +67,14 @@ typedef enum { CONTAINER_TAG_GC_SEEN = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2) } ContainerTag; +typedef enum { + // Must match to permTag() in Kotlin. + OBJECT_TAG_PERMANENT_CONTAINER = 1 << 0, + OBJECT_TAG_NONTRIVIAL_CONTAINER = 1 << 1, + // Keep in sync with immTypeInfoMask in Kotlin. + OBJECT_TAG_MASK = (1 << 2) - 1 +} ObjectTag; + typedef uint32_t container_size_t; // Header of all container objects. Contains reference counter. @@ -86,10 +89,6 @@ struct ContainerHeader { return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NORMAL; } - inline bool permanent() const { - return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT; - } - inline bool frozen() const { return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN; } @@ -102,12 +101,8 @@ struct ContainerHeader { refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_ATOMIC; } - inline bool permanentOrFrozen() const { - return tag() == CONTAINER_TAG_PERMANENT || tag() == CONTAINER_TAG_FROZEN; - } - inline bool shareable() const { - return (tag() & 1) != 0; // CONTAINER_TAG_PERMANENT || CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC + return (tag() & 1) != 0; // CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC } inline bool stack() const { @@ -228,29 +223,83 @@ struct ContainerHeader { } }; +inline bool PermanentOrFrozen(ContainerHeader* container) { + return container == nullptr || container->frozen(); +} + +inline bool Shareable(ContainerHeader* container) { + return container == nullptr || container->shareable(); +} + struct ArrayHeader; struct MetaObjHeader; +template +ALWAYS_INLINE T* setPointerBits(T* ptr, unsigned bits) { + return reinterpret_cast(reinterpret_cast(ptr) | bits); +} + +template +ALWAYS_INLINE T* clearPointerBits(T* ptr, unsigned bits) { + return reinterpret_cast(reinterpret_cast(ptr) & ~static_cast(bits)); +} + +template +ALWAYS_INLINE unsigned getPointerBits(T* ptr, unsigned bits) { + return reinterpret_cast(ptr) & static_cast(bits); +} + +template +ALWAYS_INLINE bool hasPointerBits(T* ptr, unsigned bits) { + return getPointerBits(ptr, bits) != 0; +} + +// Header for the meta-object. +struct MetaObjHeader { + // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout. + const TypeInfo* typeInfo_; + // Strong reference to the counter object. + ObjHeader* counter_; + // Container pointer. + ContainerHeader* container_; +#ifdef KONAN_OBJC_INTEROP + void* associatedObject_; +#endif + + // Flags for the object state. + int32_t flags_; +}; + // Header of every object. struct ObjHeader { TypeInfo* typeInfoOrMeta_; - ContainerHeader* container_; const TypeInfo* type_info() const { - return typeInfoOrMeta_->typeInfo_; + return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_; } bool has_meta_object() const { - return typeInfoOrMeta_ != typeInfoOrMeta_->typeInfo_; + auto* typeInfoOrMeta = clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK); + return (typeInfoOrMeta != typeInfoOrMeta->typeInfo_); } MetaObjHeader* meta_object() { return has_meta_object() ? - reinterpret_cast(typeInfoOrMeta_) : createMetaObject(&typeInfoOrMeta_); + reinterpret_cast(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)) : + createMetaObject(&typeInfoOrMeta_); + } + + void setContainer(ContainerHeader* container) { + meta_object()->container_ = container; + typeInfoOrMeta_ = setPointerBits(typeInfoOrMeta_, OBJECT_TAG_NONTRIVIAL_CONTAINER); } ContainerHeader* container() const { - return container_; + unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK); + if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) return nullptr; + return (bits & OBJECT_TAG_NONTRIVIAL_CONTAINER) != 0 ? + (reinterpret_cast(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_ : + reinterpret_cast(const_cast(this)) - 1; } // Unsafe cast to ArrayHeader. Use carefully! @@ -258,7 +307,7 @@ struct ObjHeader { const ArrayHeader* array() const { return reinterpret_cast(this); } inline bool permanent() const { - return container()->permanent(); + return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER); } static MetaObjHeader* createMetaObject(TypeInfo** location); @@ -268,14 +317,9 @@ struct ObjHeader { // Header of value type array objects. Keep layout in sync with that of object header. struct ArrayHeader { TypeInfo* typeInfoOrMeta_; - ContainerHeader* container_; const TypeInfo* type_info() const { - return typeInfoOrMeta_->typeInfo_; - } - - ContainerHeader* container() const { - return container_; + return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_; } ObjHeader* obj() { return reinterpret_cast(this); } @@ -285,19 +329,10 @@ struct ArrayHeader { uint32_t count_; }; -// Header for the meta-object. -struct MetaObjHeader { - // Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout. - const TypeInfo* typeInfo_; - // Strong reference to counter object. - ObjHeader* counter_; - // Flags for object state. - int32_t flags_; - -#ifdef KONAN_OBJC_INTEROP - void* associatedObject_; -#endif -}; +inline bool PermanentOrFrozen(ObjHeader* obj) { + auto* container = obj->container(); + return container == nullptr || container->frozen(); +} inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { // Instance size is negative. @@ -311,7 +346,6 @@ class Container { ContainerHeader* header_; void SetHeader(ObjHeader* obj, const TypeInfo* type_info) { - obj->container_ = header_; obj->typeInfoOrMeta_ = const_cast(type_info); // Take into account typeInfo's immutability for ARC strategy. if ((type_info->flags_ & TF_IMMUTABLE) != 0) @@ -393,8 +427,8 @@ class ArenaContainer { bool allocContainer(container_size_t minSize); void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) { - obj->container_ = currentChunk_->asHeader(); obj->typeInfoOrMeta_ = const_cast(typeInfo); + obj->setContainer(currentChunk_->asHeader()); // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC. } diff --git a/runtime/src/main/cpp/ObjCExport.mm b/runtime/src/main/cpp/ObjCExport.mm index 9f85804acc2..94cf984fe54 100644 --- a/runtime/src/main/cpp/ObjCExport.mm +++ b/runtime/src/main/cpp/ObjCExport.mm @@ -113,7 +113,7 @@ static void initializeClass(Class clazz); extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject); static inline id AtomicSetAssociatedObject(ObjHeader* obj, id associatedObject) { - if (!obj->container()->permanentOrFrozen()) { + if (!PermanentOrFrozen(obj)) { SetAssociatedObject(obj, associatedObject); return associatedObject; } else { diff --git a/runtime/src/main/cpp/Worker.cpp b/runtime/src/main/cpp/Worker.cpp index a941525efd9..962b53321f6 100644 --- a/runtime/src/main/cpp/Worker.cpp +++ b/runtime/src/main/cpp/Worker.cpp @@ -547,7 +547,7 @@ void Kotlin_Worker_freezeInternal(KRef object) { } KBoolean Kotlin_Worker_isFrozenInternal(KRef object) { - return object == nullptr || object->container()->permanentOrFrozen(); + return object == nullptr || PermanentOrFrozen(object); } void Kotlin_Worker_ensureNeverFrozen(KRef object) { diff --git a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt index 8f1adfdf11f..90d6544a991 100644 --- a/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt +++ b/shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt @@ -6,7 +6,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion { data class KonanAbiVersion(val version: Int) { companion object { - val CURRENT = KonanAbiVersion(4) + val CURRENT = KonanAbiVersion(5) } override fun toString() = "$version" }