azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reduce size of object header by one pointer. (#2432)

Browse Source
This commit is contained in:
Nikolay Igotti
2018-12-03 13:50:56 +03:00
committed by GitHub
parent c6ac807034
commit 9e3a68f835
9 changed files with 162 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/CodeGenerator.kt
+8 -3
View File
@@ -9,7 +9,6 @@ package org.jetbrains.kotlin.backend.konan.llvm
import kotlinx.cinterop.* import kotlinx.cinterop.*
import llvm.* import llvm.*
import org.jetbrains.kotlin.backend.konan.* import org.jetbrains.kotlin.backend.konan.*
import org.jetbrains.kotlin.backend.konan.descriptors.*
import org.jetbrains.kotlin.backend.konan.descriptors.isInterface import org.jetbrains.kotlin.backend.konan.descriptors.isInterface
import org.jetbrains.kotlin.backend.konan.irasdescriptors.* import org.jetbrains.kotlin.backend.konan.irasdescriptors.*
import org.jetbrains.kotlin.backend.konan.irasdescriptors.ClassConstructorDescriptor import org.jetbrains.kotlin.backend.konan.irasdescriptors.ClassConstructorDescriptor
@@ -28,6 +27,8 @@ internal class CodeGenerator(override val context: Context) : ContextUtils {
fun llvmFunction(function: FunctionDescriptor): LLVMValueRef = function.llvmFunction fun llvmFunction(function: FunctionDescriptor): LLVMValueRef = function.llvmFunction
val intPtrType = LLVMIntPtrType(llvmTargetData)!! val intPtrType = LLVMIntPtrType(llvmTargetData)!!
internal val immOneIntPtrType = LLVMConstInt(intPtrType, 1, 1)!! internal val immOneIntPtrType = LLVMConstInt(intPtrType, 1, 1)!!
// Keep in sync with OBJECT_TAG_MASK in C++.
internal val immTypeInfoMask = LLVMConstNot(LLVMConstInt(intPtrType, 3, 0)!!)!!
//-------------------------------------------------------------------------// //-------------------------------------------------------------------------//
@@ -624,7 +625,11 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
call(context.llvm.getObjCKotlinTypeInfo, listOf(receiver)) call(context.llvm.getObjCKotlinTypeInfo, listOf(receiver))
} else { } else {
val typeInfoOrMetaPtr = structGep(receiver, 0 /* typeInfoOrMeta_ */) val typeInfoOrMetaPtr = structGep(receiver, 0 /* typeInfoOrMeta_ */)
val typeInfoOrMeta = load(typeInfoOrMetaPtr) val typeInfoOrMetaWithFlags = load(typeInfoOrMetaPtr)
// Clear two lower bits.
val typeInfoOrMetaWithFlagsRaw = ptrToInt(typeInfoOrMetaWithFlags, codegen.intPtrType)
val typeInfoOrMetaRaw = and(typeInfoOrMetaWithFlagsRaw, codegen.immTypeInfoMask)
val typeInfoOrMeta = intToPtr(typeInfoOrMetaRaw, kTypeInfoPtr)
val typeInfoPtrPtr = structGep(typeInfoOrMeta, 0 /* typeInfo */) val typeInfoPtrPtr = structGep(typeInfoOrMeta, 0 /* typeInfo */)
load(typeInfoPtrPtr) load(typeInfoPtrPtr)
} }
@@ -634,7 +639,7 @@ internal class FunctionGenerationContext(val function: LLVMValueRef,
/* /*
* Resolve owner of the call with special handling of Any methods: * Resolve owner of the call with special handling of Any methods:
* if toString/eq/hc is invoked on an interface instance, we resolve * if toString/eq/hc is invoked on an interface instance, we resolve
* owner as Any and dispatch it via vtable * owner as Any and dispatch it via vtable.
*/ */
val anyMethod = (descriptor as SimpleFunctionDescriptor).findOverriddenMethodOfAny() val anyMethod = (descriptor as SimpleFunctionDescriptor).findOverriddenMethodOfAny()
val owner = (anyMethod ?: descriptor).containingDeclaration as ClassDescriptor val owner = (anyMethod ?: descriptor).containingDeclaration as ClassDescriptor
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/ContextUtils.kt
-2
View File
@@ -457,8 +457,6 @@ internal class Llvm(val context: Context, val llvmModule: LLVMModuleRef) {
origin = context.standardLlvmSymbolsOrigin origin = context.standardLlvmSymbolsOrigin
) )
val staticContainer = importRtGlobal("theStaticObjectsContainer")
val memsetFunction = importMemset() val memsetFunction = importMemset()
val usedFunctions = mutableListOf<LLVMValueRef>() val usedFunctions = mutableListOf<LLVMValueRef>()
backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt
+13 -8
View File
@@ -5,10 +5,8 @@
package org.jetbrains.kotlin.backend.konan.llvm package org.jetbrains.kotlin.backend.konan.llvm
import llvm.LLVMLinkage import kotlinx.cinterop.cValuesOf
import llvm.LLVMSetLinkage import llvm.*
import llvm.LLVMTypeRef
import llvm.LLVMValueRef
import org.jetbrains.kotlin.backend.konan.irasdescriptors.fqNameSafe import org.jetbrains.kotlin.backend.konan.irasdescriptors.fqNameSafe
import org.jetbrains.kotlin.backend.konan.irasdescriptors.llvmSymbolOrigin import org.jetbrains.kotlin.backend.konan.irasdescriptors.llvmSymbolOrigin
import org.jetbrains.kotlin.descriptors.ClassDescriptor import org.jetbrains.kotlin.descriptors.ClassDescriptor
@@ -17,16 +15,23 @@ import org.jetbrains.kotlin.ir.declarations.IrClass
import org.jetbrains.kotlin.name.FqName import org.jetbrains.kotlin.name.FqName
import org.jetbrains.kotlin.name.Name import org.jetbrains.kotlin.name.Name
private fun ConstPointer.add(index: Int): ConstPointer {
return constPointer(LLVMConstGEP(llvm, cValuesOf(Int32(index).llvm), 1)!!)
}
// Must match OBJECT_TAG_PERMANENT_CONTAINER in C++.
private fun StaticData.permanentTag(typeInfo: ConstPointer): ConstPointer {
// Only pointer arithmetic via GEP works on constant pointers in LLVM.
return typeInfo.bitcast(int8TypePtr).add(1).bitcast(kTypeInfoPtr)
}
private fun StaticData.objHeader(typeInfo: ConstPointer): Struct { private fun StaticData.objHeader(typeInfo: ConstPointer): Struct {
val container = constValue(context.llvm.staticContainer) return Struct(runtime.objHeaderType, permanentTag(typeInfo))
return Struct(runtime.objHeaderType, typeInfo, container)
} }
private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct { private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct {
assert (length >= 0) assert (length >= 0)
val container = constValue(context.llvm.staticContainer) return Struct(runtime.arrayHeaderType, permanentTag(typeInfo), Int32(length))
return Struct(runtime.arrayHeaderType, typeInfo, container, Int32(length))
} }
internal fun StaticData.createKotlinStringLiteral(value: String): ConstPointer { internal fun StaticData.createKotlinStringLiteral(value: String): ConstPointer {
runtime/src/main/cpp/Atomic.cpp
+1 -1
View File
@@ -168,7 +168,7 @@ KNativePtr Kotlin_AtomicNativePtr_get(KRef thiz) {
} }
void Kotlin_AtomicReference_checkIfFrozen(KRef value) { void Kotlin_AtomicReference_checkIfFrozen(KRef value) {
if (value != nullptr && !value->container()->permanentOrFrozen()) { if (value != nullptr && !PermanentOrFrozen(value)) {
ThrowInvalidMutabilityException(value); ThrowInvalidMutabilityException(value);
} }
} }
runtime/src/main/cpp/Memory.cpp
+60 -61
View File
@@ -180,10 +180,10 @@ public:
} }
static int toIndex(const ContainerHeader* header) { static int toIndex(const ContainerHeader* header) {
if (header == nullptr) return 2; // permanent.
switch (header->tag()) { switch (header->tag()) {
case CONTAINER_TAG_NORMAL : return 0; case CONTAINER_TAG_NORMAL : return 0;
case CONTAINER_TAG_STACK : return 1; case CONTAINER_TAG_STACK : return 1;
case CONTAINER_TAG_PERMANENT: return 2;
case CONTAINER_TAG_FROZEN: return 3; case CONTAINER_TAG_FROZEN: return 3;
} }
RuntimeAssert(false, "unknown container type"); RuntimeAssert(false, "unknown container type");
@@ -381,15 +381,15 @@ THREAD_LOCAL_VARIABLE MemoryState* memoryState = nullptr;
constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**); constexpr int kFrameOverlaySlots = sizeof(FrameOverlay) / sizeof(ObjHeader**);
inline bool isFreeable(const ContainerHeader* header) { inline bool isFreeable(const ContainerHeader* header) {
return header->tag() < CONTAINER_TAG_PERMANENT; return header != nullptr && header->tag() != CONTAINER_TAG_STACK;
} }
inline bool isArena(const ContainerHeader* header) { inline bool isArena(const ContainerHeader* header) {
return header->stack(); return header != nullptr && header->stack();
} }
inline bool isAggregatingFrozenContainer(const ContainerHeader* header) { inline bool isAggregatingFrozenContainer(const ContainerHeader* header) {
return header->frozen() && header->objectCount() > 1; return header != nullptr && header->frozen() && header->objectCount() > 1;
} }
inline container_size_t alignUp(container_size_t size, int alignment) { inline container_size_t alignUp(container_size_t size, int alignment) {
@@ -446,8 +446,8 @@ void KRefSharedHolder::verifyRefOwner() const {
// Initialized runtime is required to throw the exception below // Initialized runtime is required to throw the exception below
// or to provide proper execution context for shared objects: // or to provide proper execution context for shared objects:
if (memoryState == nullptr) Kotlin_initRuntimeIfNeeded(); if (memoryState == nullptr) Kotlin_initRuntimeIfNeeded();
auto* container = obj_->container();
if (!obj_->container()->shareable()) { if (!Shareable(container)) {
// TODO: add some info about the owner. // TODO: add some info about the owner.
ThrowIllegalObjectSharingException(obj_->type_info(), obj_); ThrowIllegalObjectSharingException(obj_->type_info(), obj_);
} }
@@ -456,13 +456,6 @@ void KRefSharedHolder::verifyRefOwner() const {
extern "C" { extern "C" {
// Ensure LLVM never throws theStaticObjectsContainer away.
// TODO: although practically const, marking it as such makes LLVM crazy, fix it.
RUNTIME_USED ContainerHeader theStaticObjectsContainer = {
CONTAINER_TAG_PERMANENT | CONTAINER_TAG_INCREMENT,
0 /* Object count */
};
void objc_release(void* ptr); void objc_release(void* ptr);
void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject); void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
RUNTIME_NORETURN void ThrowFreezingException(KRef toFreeze, KRef blocker); RUNTIME_NORETURN void ThrowFreezingException(KRef toFreeze, KRef blocker);
@@ -639,9 +632,9 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet*
header->refCount_ >> CONTAINER_TAG_SHIFT) header->refCount_ >> CONTAINER_TAG_SHIFT)
seen->insert(header); seen->insert(header);
traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) { traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) {
auto child = ref->container(); auto* child = ref->container();
RuntimeAssert(!isArena(child), "A reference to local object is encountered"); RuntimeAssert(!isArena(child), "A reference to local object is encountered");
if (!child->permanent() && (seen->count(child) == 0)) { if (child != nullptr && (seen->count(child) == 0)) {
dumpWorker(prefix, child, seen); dumpWorker(prefix, child, seen);
} }
}); });
@@ -650,9 +643,8 @@ void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet*
void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
ContainerHeaderSet seen; ContainerHeaderSet seen;
for (auto container : *roots) { for (auto container : *roots) {
MEMORY_LOG("%p: %s%s%s\n", container, MEMORY_LOG("%p: %s%s\n", container,
container->frozen() ? "frozen " : "", container->frozen() ? "frozen " : "",
container->permanent() ? "permanent " : "",
container->stack() ? "stack " : "") container->stack() ? "stack " : "")
dumpWorker(prefix, container, &seen); dumpWorker(prefix, container, &seen);
} }
@@ -696,8 +688,7 @@ void MarkGray(ContainerHeader* start) {
traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
auto childContainer = ref->container(); auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!Shareable(childContainer)) {
if (!childContainer->shareable()) {
childContainer->decRefCount<false>(); childContainer->decRefCount<false>();
toVisit.push_front(childContainer); toVisit.push_front(childContainer);
} }
@@ -723,7 +714,7 @@ void ScanBlack(ContainerHeader* start) {
traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
auto childContainer = ref->container(); auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!childContainer->shareable()) { if (!Shareable(childContainer)) {
childContainer->incRefCount<false>(); childContainer->incRefCount<false>();
if (useColor) { if (useColor) {
int color = childContainer->color(); int color = childContainer->color();
@@ -802,7 +793,7 @@ void Scan(ContainerHeader* start) {
traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) { traverseContainerReferredObjects(container, [&toVisit](ObjHeader* ref) {
auto* childContainer = ref->container(); auto* childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!childContainer->shareable()) { if (!Shareable(childContainer)) {
toVisit.push_front(childContainer); toVisit.push_front(childContainer);
} }
}); });
@@ -823,7 +814,7 @@ void CollectWhite(MemoryState* state, ContainerHeader* start) {
if (ref == nullptr) return; if (ref == nullptr) return;
auto* childContainer = ref->container(); auto* childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (childContainer->shareable()) { if (Shareable(childContainer)) {
UpdateRef(location, nullptr); UpdateRef(location, nullptr);
} else { } else {
toVisit.push_front(childContainer); toVisit.push_front(childContainer);
@@ -840,7 +831,6 @@ inline void AddRef(ContainerHeader* header) {
// (non-escaping stack objects, constant objects). // (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) { switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK: case CONTAINER_TAG_STACK:
case CONTAINER_TAG_PERMANENT:
break; break;
case CONTAINER_TAG_NORMAL: case CONTAINER_TAG_NORMAL:
IncrementRC</* Atomic = */ false>(header); IncrementRC</* Atomic = */ false>(header);
@@ -856,7 +846,6 @@ inline void ReleaseRef(ContainerHeader* header) {
// Looking at container type we may want to skip ReleaseRef() totally // Looking at container type we may want to skip ReleaseRef() totally
// (non-escaping stack objects, constant objects). // (non-escaping stack objects, constant objects).
switch (header->tag()) { switch (header->tag()) {
case CONTAINER_TAG_PERMANENT:
case CONTAINER_TAG_STACK: case CONTAINER_TAG_STACK:
break; break;
case CONTAINER_TAG_NORMAL: case CONTAINER_TAG_NORMAL:
@@ -901,6 +890,7 @@ inline size_t containerSize(const ContainerHeader* container) {
MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) { MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
MetaObjHeader* meta = konanConstructInstance<MetaObjHeader>(); MetaObjHeader* meta = konanConstructInstance<MetaObjHeader>();
TypeInfo* typeInfo = *location; TypeInfo* typeInfo = *location;
RuntimeCheck(!hasPointerBits(typeInfo, OBJECT_TAG_MASK), "Object must not be tagged");
meta->typeInfo_ = typeInfo; meta->typeInfo_ = typeInfo;
#if KONAN_NO_THREADS #if KONAN_NO_THREADS
*location = reinterpret_cast<TypeInfo*>(meta); *location = reinterpret_cast<TypeInfo*>(meta);
@@ -916,7 +906,7 @@ MetaObjHeader* ObjHeader::createMetaObject(TypeInfo** location) {
} }
void ObjHeader::destroyMetaObject(TypeInfo** location) { void ObjHeader::destroyMetaObject(TypeInfo** location) {
MetaObjHeader* meta = *(reinterpret_cast<MetaObjHeader**>(location)); MetaObjHeader* meta = clearPointerBits(*(reinterpret_cast<MetaObjHeader**>(location)), OBJECT_TAG_MASK);
*const_cast<const TypeInfo**>(location) = meta->typeInfo_; *const_cast<const TypeInfo**>(location) = meta->typeInfo_;
if (meta->counter_ != nullptr) { if (meta->counter_ != nullptr) {
WeakReferenceCounterClear(meta->counter_); WeakReferenceCounterClear(meta->counter_);
@@ -947,13 +937,13 @@ ContainerHeader* AllocContainer(size_t size) {
ContainerHeader* AllocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& containers) { ContainerHeader* AllocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& containers) {
auto componentSize = containers.size(); auto componentSize = containers.size();
auto superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize); auto* superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize);
auto place = reinterpret_cast<ContainerHeader**>(superContainer + 1); auto* place = reinterpret_cast<ContainerHeader**>(superContainer + 1);
for (auto* container : containers) { for (auto* container : containers) {
*place++ = container; *place++ = container;
// Set link to the new container. // Set link to the new container.
auto obj = reinterpret_cast<ObjHeader*>(container + 1); auto* obj = reinterpret_cast<ObjHeader*>(container + 1);
obj->container_ = superContainer; obj->setContainer(superContainer);
MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer); MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer);
} }
superContainer->setObjectCount(componentSize); superContainer->setObjectCount(componentSize);
@@ -962,7 +952,7 @@ ContainerHeader* AllocAggregatingFrozenContainer(KStdVector<ContainerHeader*>& c
} }
void FreeAggregatingFrozenContainer(ContainerHeader* container) { void FreeAggregatingFrozenContainer(ContainerHeader* container) {
auto state = memoryState; auto* state = memoryState;
RuntimeAssert(isAggregatingFrozenContainer(container), "expected fictitious frozen container"); RuntimeAssert(isAggregatingFrozenContainer(container), "expected fictitious frozen container");
MEMORY_LOG("%p is fictitious frozen container\n", container); MEMORY_LOG("%p is fictitious frozen container\n", container);
RuntimeAssert(!container->buffered(), "frozen objects must not participate in GC") RuntimeAssert(!container->buffered(), "frozen objects must not participate in GC")
@@ -985,7 +975,7 @@ void FreeAggregatingFrozenContainer(ContainerHeader* container) {
} }
void FreeContainer(ContainerHeader* container) { void FreeContainer(ContainerHeader* container) {
RuntimeAssert(!container->permanent(), "this kind of container shalln't be freed"); RuntimeAssert(container != nullptr, "this kind of container shalln't be freed");
auto state = memoryState; auto state = memoryState;
CONTAINER_FREE_EVENT(state, container) CONTAINER_FREE_EVENT(state, container)
@@ -1138,13 +1128,19 @@ ArrayHeader* ArenaContainer::PlaceArray(const TypeInfo* type_info, uint32_t coun
} }
inline void AddRef(const ObjHeader* object) { inline void AddRef(const ObjHeader* object) {
MEMORY_LOG("AddRef on %p in %p\n", object, object->container()) auto* container = object->container();
AddRef(object->container()); if (container != nullptr) {
MEMORY_LOG("AddRef on %p in %p\n", object, container)
AddRef(container);
}
} }
inline void ReleaseRef(const ObjHeader* object) { inline void ReleaseRef(const ObjHeader* object) {
MEMORY_LOG("ReleaseRef on %p in %p\n", object, object->container()) auto* container = object->container();
ReleaseRef(object->container()); if (container != nullptr) {
MEMORY_LOG("ReleaseRef on %p in %p\n", object, container)
ReleaseRef(container);
}
} }
void AddRefFromAssociatedObject(const ObjHeader* object) { void AddRefFromAssociatedObject(const ObjHeader* object) {
@@ -1162,10 +1158,6 @@ MemoryState* InitMemory() {
== ==
offsetof(ObjHeader, typeInfoOrMeta_), offsetof(ObjHeader, typeInfoOrMeta_),
"Layout mismatch"); "Layout mismatch");
RuntimeAssert(offsetof(ArrayHeader, container_)
==
offsetof(ObjHeader , container_),
"Layout mismatch");
RuntimeAssert(offsetof(TypeInfo, typeInfo_) RuntimeAssert(offsetof(TypeInfo, typeInfo_)
== ==
offsetof(MetaObjHeader, typeInfo_), offsetof(MetaObjHeader, typeInfo_),
@@ -1357,7 +1349,7 @@ ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot)
ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) { ObjHeader** GetParamSlotIfArena(ObjHeader* param, ObjHeader** localSlot) {
if (param == nullptr) return localSlot; if (param == nullptr) return localSlot;
auto container = param->container(); auto container = param->container();
if ((container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK) if (container == nullptr || (container->refCount_ & CONTAINER_TAG_MASK) != CONTAINER_TAG_STACK)
return localSlot; return localSlot;
auto chunk = reinterpret_cast<ContainerChunk*>(container) - 1; auto chunk = reinterpret_cast<ContainerChunk*>(container) - 1;
return reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(&chunk->arena) | ARENA_BIT); return reinterpret_cast<ObjHeader**>(reinterpret_cast<uintptr_t>(&chunk->arena) | ARENA_BIT);
@@ -1556,7 +1548,7 @@ KInt Kotlin_native_internal_GC_getThreshold(KRef) {
KNativePtr CreateStablePointer(KRef any) { KNativePtr CreateStablePointer(KRef any) {
if (any == nullptr) return nullptr; if (any == nullptr) return nullptr;
AddRef(any->container()); AddRef(any);
return reinterpret_cast<KNativePtr>(any); return reinterpret_cast<KNativePtr>(any);
} }
@@ -1593,7 +1585,7 @@ bool hasExternalRefs(ContainerHeader* start, ContainerHeaderSet* visited) {
if (container->refCount() != 0) return true; if (container->refCount() != 0) return true;
traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) { traverseContainerReferredObjects(container, [&toVisit, visited](ObjHeader* ref) {
auto* child = ref->container(); auto* child = ref->container();
if (!child->shareable() && (visited->count(child) == 0)) { if (!Shareable(child) && (visited->count(child) == 0)) {
toVisit.push_front(child); toVisit.push_front(child);
} }
}); });
@@ -1608,7 +1600,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
auto state = memoryState; auto state = memoryState;
auto* container = root->container(); auto* container = root->container();
if (container->frozen()) if (container == nullptr || container->frozen())
// We assume, that frozen objects can be safely passed and are already removed // We assume, that frozen objects can be safely passed and are already removed
// GC candidate list. // GC candidate list.
return true; return true;
@@ -1617,7 +1609,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
if (!checked) { if (!checked) {
hasExternalRefs(container, &visited); hasExternalRefs(container, &visited);
} else { } else {
if (!container->shareable()) { if (!Shareable(container)) {
container->decRefCount<false>(); container->decRefCount<false>();
MarkGray<false>(container); MarkGray<false>(container);
auto bad = hasExternalRefs(container, &visited); auto bad = hasExternalRefs(container, &visited);
@@ -1661,7 +1653,7 @@ void depthFirstTraversal(ContainerHeader* container, bool* hasCycles,
return; return;
} }
ContainerHeader* objContainer = obj->container(); ContainerHeader* objContainer = obj->container();
if (!objContainer->shareable()) { if (!Shareable(objContainer)) {
// Marked GREY, there's cycle. // Marked GREY, there's cycle.
if (objContainer->seen()) *hasCycles = true; if (objContainer->seen()) *hasCycles = true;
@@ -1704,7 +1696,7 @@ void freezeAcyclic(ContainerHeader* rootContainer) {
current->freeze(); current->freeze();
traverseContainerReferredObjects(current, [current, &queue](ObjHeader* obj) { traverseContainerReferredObjects(current, [current, &queue](ObjHeader* obj) {
ContainerHeader* objContainer = obj->container(); ContainerHeader* objContainer = obj->container();
if (!objContainer->shareable()) { if (!Shareable(objContainer)) {
if (objContainer->marked()) if (objContainer->marked())
queue.push_back(objContainer); queue.push_back(objContainer);
} }
@@ -1723,7 +1715,7 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
reversedEdges.emplace(current, KStdVector<ContainerHeader*>(0)); reversedEdges.emplace(current, KStdVector<ContainerHeader*>(0));
traverseContainerReferredObjects(current, [current, &queue, &reversedEdges](ObjHeader* obj) { traverseContainerReferredObjects(current, [current, &queue, &reversedEdges](ObjHeader* obj) {
ContainerHeader* objContainer = obj->container(); ContainerHeader* objContainer = obj->container();
if (!objContainer->shareable()) { if (!Shareable(objContainer)) {
if (objContainer->marked()) if (objContainer->marked())
queue.push_back(objContainer); queue.push_back(objContainer);
reversedEdges.emplace(objContainer, KStdVector<ContainerHeader*>(0)).first->second.push_back(current); reversedEdges.emplace(objContainer, KStdVector<ContainerHeader*>(0)).first->second.push_back(current);
@@ -1755,14 +1747,11 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
for (auto* container : component) { for (auto* container : component) {
totalCount += container->refCount(); totalCount += container->refCount();
traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) { traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) {
if (!obj->container()->shareable()) auto* container = obj->container();
if (!Shareable(container))
++internalRefsCount; ++internalRefsCount;
}); });
} }
// Create fictitious container for the whole component.
auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component);
// Don't count internal references.
superContainer->setRefCount(totalCount - internalRefsCount);
// Freeze component. // Freeze component.
for (auto* container : component) { for (auto* container : component) {
@@ -1771,7 +1760,14 @@ void freezeCyclic(ContainerHeader* rootContainer, const KStdVector<ContainerHead
// Note, that once object is frozen, it could be concurrently accessed, so // Note, that once object is frozen, it could be concurrently accessed, so
// color and similar attributes shall not be used. // color and similar attributes shall not be used.
container->freeze(); container->freeze();
// We set refcount of original container to zero, so that it is seen as such after removal
// meta-object, where aggregating container is stored.
container->setRefCount(0);
} }
// Create fictitious container for the whole component.
auto superContainer = component.size() == 1 ? component[0] : AllocAggregatingFrozenContainer(component);
// Don't count internal references.
superContainer->setRefCount(totalCount - internalRefsCount);
} }
} }
@@ -1803,7 +1799,7 @@ void FreezeSubgraph(ObjHeader* root) {
// First check that passed object graph has no cycles. // First check that passed object graph has no cycles.
// If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir. // If there are cycles - run graph condensation on cyclic graphs using Kosoraju-Sharir.
ContainerHeader* rootContainer = root->container(); ContainerHeader* rootContainer = root->container();
if (rootContainer->shareable()) return; if (Shareable(rootContainer)) return;
// Do DFS cycle detection. // Do DFS cycle detection.
bool hasCycles = false; bool hasCycles = false;
@@ -1836,7 +1832,8 @@ void FreezeSubgraph(ObjHeader* root) {
// This function is called from field mutators to check if object's header is frozen. // This function is called from field mutators to check if object's header is frozen.
// If object is frozen, an exception is thrown. // If object is frozen, an exception is thrown.
void MutationCheck(ObjHeader* obj) { void MutationCheck(ObjHeader* obj) {
if (obj->container()->frozen()) ThrowInvalidMutabilityException(obj); auto* container = obj->container();
if (container != nullptr && container->frozen()) ThrowInvalidMutabilityException(obj);
} }
OBJ_GETTER(SwapRefLocked, OBJ_GETTER(SwapRefLocked,
@@ -1881,7 +1878,8 @@ OBJ_GETTER(ReadRefLocked, ObjHeader** location, int32_t* spinlock) {
} }
void EnsureNeverFrozen(ObjHeader* object) { void EnsureNeverFrozen(ObjHeader* object) {
if (object->container()->frozen()) auto* container = object->container();
if (container == nullptr || container->frozen())
ThrowFreezingException(object, object); ThrowFreezingException(object, object);
// TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency, // TODO: note, that this API could not not be called on frozen objects, so no need to care much about concurrency,
// although there's subtle race with case, where other thread freezes the same object after check. // although there's subtle race with case, where other thread freezes the same object after check.
@@ -1889,15 +1887,16 @@ void EnsureNeverFrozen(ObjHeader* object) {
} }
KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) { KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what) {
RuntimeAssert(where->container()->frozen(), "Must be used on frozen objects only"); RuntimeAssert(where->container() != nullptr && where->container()->frozen(), "Must be used on frozen objects only");
RuntimeAssert(what == nullptr || what->container()->permanentOrFrozen(), RuntimeAssert(what == nullptr || PermanentOrFrozen(what),
"Must be used with an immutable value"); "Must be used with an immutable value");
if (what != nullptr) { if (what != nullptr) {
// Now we check that `where` is not reachable from `what`. // Now we check that `where` is not reachable from `what`.
// As we cannot modify objects while traversing, instead we remember all seen objects in a set. // As we cannot modify objects while traversing, instead we remember all seen objects in a set.
KStdUnorderedSet<ContainerHeader*> seen; KStdUnorderedSet<ContainerHeader*> seen;
KStdDeque<ContainerHeader*> queue; KStdDeque<ContainerHeader*> queue;
queue.push_back(what->container()); if (what->container() != nullptr)
queue.push_back(what->container());
bool acyclic = true; bool acyclic = true;
while (!queue.empty() && acyclic) { while (!queue.empty() && acyclic) {
ContainerHeader* current = queue.front(); ContainerHeader* current = queue.front();
@@ -1915,7 +1914,7 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what
acyclic = false; acyclic = false;
} else { } else {
auto* objContainer = obj->container(); auto* objContainer = obj->container();
if (seen.count(objContainer) == 0) if (objContainer != nullptr && seen.count(objContainer) == 0)
queue.push_back(objContainer); queue.push_back(objContainer);
} }
}); });
@@ -1930,8 +1929,8 @@ KBoolean Konan_ensureAcyclicAndSet(ObjHeader* where, KInt index, ObjHeader* what
} }
void Kotlin_Any_share(ObjHeader* obj) { void Kotlin_Any_share(ObjHeader* obj) {
auto container = obj->container(); auto* container = obj->container();
if (container->shareable()) return; if (Shareable(container)) return;
RuntimeCheck(container->objectCount() == 1, "Must be a single object container"); RuntimeCheck(container->objectCount() == 1, "Must be a single object container");
container->makeShareable(); container->makeShareable();
} }
runtime/src/main/cpp/Memory.h
+77 -43
View File
@@ -31,19 +31,14 @@ typedef enum {
// Stack container, no need to free, children cleanup still shall be there. // Stack container, no need to free, children cleanup still shall be there.
CONTAINER_TAG_STACK = 2, CONTAINER_TAG_STACK = 2,
// Atomic container, reference counter is atomically updated. // Atomic container, reference counter is atomically updated.
CONTAINER_TAG_ATOMIC = 5 | 1, // shareable CONTAINER_TAG_ATOMIC = 3 | 1, // shareable
// Those container tags shall not be refcounted.
// Permanent container, cannot refer to non-permanent containers, so no need to cleanup those.
// Please check isFreeable() if changing the numeric value.
CONTAINER_TAG_PERMANENT = 7 | 1, // shareable
// Shift to get actual counter. // Shift to get actual counter.
CONTAINER_TAG_SHIFT = 3, CONTAINER_TAG_SHIFT = 2,
// Actual value to increment/decrement container by. Tag is in lower bits. // Actual value to increment/decrement container by. Tag is in lower bits.
CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT, CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
// Mask for container type. // Mask for container type.
CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1, CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
// Those bit masks are applied to objectCount_ field.
// Shift to get actual object count. // Shift to get actual object count.
CONTAINER_TAG_GC_SHIFT = 6, CONTAINER_TAG_GC_SHIFT = 6,
CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT, CONTAINER_TAG_GC_INCREMENT = 1 << CONTAINER_TAG_GC_SHIFT,
@@ -72,6 +67,14 @@ typedef enum {
CONTAINER_TAG_GC_SEEN = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2) CONTAINER_TAG_GC_SEEN = 1 << (CONTAINER_TAG_COLOR_SHIFT + 2)
} ContainerTag; } ContainerTag;
typedef enum {
// Must match to permTag() in Kotlin.
OBJECT_TAG_PERMANENT_CONTAINER = 1 << 0,
OBJECT_TAG_NONTRIVIAL_CONTAINER = 1 << 1,
// Keep in sync with immTypeInfoMask in Kotlin.
OBJECT_TAG_MASK = (1 << 2) - 1
} ObjectTag;
typedef uint32_t container_size_t; typedef uint32_t container_size_t;
// Header of all container objects. Contains reference counter. // Header of all container objects. Contains reference counter.
@@ -86,10 +89,6 @@ struct ContainerHeader {
return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NORMAL; return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NORMAL;
} }
inline bool permanent() const {
return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
}
inline bool frozen() const { inline bool frozen() const {
return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN; return (refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_FROZEN;
} }
@@ -102,12 +101,8 @@ struct ContainerHeader {
refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_ATOMIC; refCount_ = (refCount_ & ~CONTAINER_TAG_MASK) | CONTAINER_TAG_ATOMIC;
} }
inline bool permanentOrFrozen() const {
return tag() == CONTAINER_TAG_PERMANENT || tag() == CONTAINER_TAG_FROZEN;
}
inline bool shareable() const { inline bool shareable() const {
return (tag() & 1) != 0; // CONTAINER_TAG_PERMANENT || CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC return (tag() & 1) != 0; // CONTAINER_TAG_FROZEN || CONTAINER_TAG_ATOMIC
} }
inline bool stack() const { inline bool stack() const {
@@ -228,29 +223,83 @@ struct ContainerHeader {
} }
}; };
inline bool PermanentOrFrozen(ContainerHeader* container) {
return container == nullptr || container->frozen();
}
inline bool Shareable(ContainerHeader* container) {
return container == nullptr || container->shareable();
}
struct ArrayHeader; struct ArrayHeader;
struct MetaObjHeader; struct MetaObjHeader;
template <typename T>
ALWAYS_INLINE T* setPointerBits(T* ptr, unsigned bits) {
return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) | bits);
}
template <typename T>
ALWAYS_INLINE T* clearPointerBits(T* ptr, unsigned bits) {
return reinterpret_cast<T*>(reinterpret_cast<uintptr_t>(ptr) & ~static_cast<uintptr_t>(bits));
}
template <typename T>
ALWAYS_INLINE unsigned getPointerBits(T* ptr, unsigned bits) {
return reinterpret_cast<uintptr_t>(ptr) & static_cast<uintptr_t>(bits);
}
template <typename T>
ALWAYS_INLINE bool hasPointerBits(T* ptr, unsigned bits) {
return getPointerBits(ptr, bits) != 0;
}
// Header for the meta-object.
struct MetaObjHeader {
// Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout.
const TypeInfo* typeInfo_;
// Strong reference to the counter object.
ObjHeader* counter_;
// Container pointer.
ContainerHeader* container_;
#ifdef KONAN_OBJC_INTEROP
void* associatedObject_;
#endif
// Flags for the object state.
int32_t flags_;
};
// Header of every object. // Header of every object.
struct ObjHeader { struct ObjHeader {
TypeInfo* typeInfoOrMeta_; TypeInfo* typeInfoOrMeta_;
ContainerHeader* container_;
const TypeInfo* type_info() const { const TypeInfo* type_info() const {
return typeInfoOrMeta_->typeInfo_; return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
} }
bool has_meta_object() const { bool has_meta_object() const {
return typeInfoOrMeta_ != typeInfoOrMeta_->typeInfo_; auto* typeInfoOrMeta = clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
return (typeInfoOrMeta != typeInfoOrMeta->typeInfo_);
} }
MetaObjHeader* meta_object() { MetaObjHeader* meta_object() {
return has_meta_object() ? return has_meta_object() ?
reinterpret_cast<MetaObjHeader*>(typeInfoOrMeta_) : createMetaObject(&typeInfoOrMeta_); reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)) :
createMetaObject(&typeInfoOrMeta_);
}
void setContainer(ContainerHeader* container) {
meta_object()->container_ = container;
typeInfoOrMeta_ = setPointerBits(typeInfoOrMeta_, OBJECT_TAG_NONTRIVIAL_CONTAINER);
} }
ContainerHeader* container() const { ContainerHeader* container() const {
return container_; unsigned bits = getPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK);
if ((bits & OBJECT_TAG_PERMANENT_CONTAINER) != 0) return nullptr;
return (bits & OBJECT_TAG_NONTRIVIAL_CONTAINER) != 0 ?
(reinterpret_cast<MetaObjHeader*>(clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)))->container_ :
reinterpret_cast<ContainerHeader*>(const_cast<ObjHeader*>(this)) - 1;
} }
// Unsafe cast to ArrayHeader. Use carefully! // Unsafe cast to ArrayHeader. Use carefully!
@@ -258,7 +307,7 @@ struct ObjHeader {
const ArrayHeader* array() const { return reinterpret_cast<const ArrayHeader*>(this); } const ArrayHeader* array() const { return reinterpret_cast<const ArrayHeader*>(this); }
inline bool permanent() const { inline bool permanent() const {
return container()->permanent(); return hasPointerBits(typeInfoOrMeta_, OBJECT_TAG_PERMANENT_CONTAINER);
} }
static MetaObjHeader* createMetaObject(TypeInfo** location); static MetaObjHeader* createMetaObject(TypeInfo** location);
@@ -268,14 +317,9 @@ struct ObjHeader {
// Header of value type array objects. Keep layout in sync with that of object header. // Header of value type array objects. Keep layout in sync with that of object header.
struct ArrayHeader { struct ArrayHeader {
TypeInfo* typeInfoOrMeta_; TypeInfo* typeInfoOrMeta_;
ContainerHeader* container_;
const TypeInfo* type_info() const { const TypeInfo* type_info() const {
return typeInfoOrMeta_->typeInfo_; return clearPointerBits(typeInfoOrMeta_, OBJECT_TAG_MASK)->typeInfo_;
}
ContainerHeader* container() const {
return container_;
} }
ObjHeader* obj() { return reinterpret_cast<ObjHeader*>(this); } ObjHeader* obj() { return reinterpret_cast<ObjHeader*>(this); }
@@ -285,19 +329,10 @@ struct ArrayHeader {
uint32_t count_; uint32_t count_;
}; };
// Header for the meta-object. inline bool PermanentOrFrozen(ObjHeader* obj) {
struct MetaObjHeader { auto* container = obj->container();
// Pointer to the type info. Must be first, to match ArrayHeader and ObjHeader layout. return container == nullptr || container->frozen();
const TypeInfo* typeInfo_; }
// Strong reference to counter object.
ObjHeader* counter_;
// Flags for object state.
int32_t flags_;
#ifdef KONAN_OBJC_INTEROP
void* associatedObject_;
#endif
};
inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
// Instance size is negative. // Instance size is negative.
@@ -311,7 +346,6 @@ class Container {
ContainerHeader* header_; ContainerHeader* header_;
void SetHeader(ObjHeader* obj, const TypeInfo* type_info) { void SetHeader(ObjHeader* obj, const TypeInfo* type_info) {
obj->container_ = header_;
obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(type_info); obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(type_info);
// Take into account typeInfo's immutability for ARC strategy. // Take into account typeInfo's immutability for ARC strategy.
if ((type_info->flags_ & TF_IMMUTABLE) != 0) if ((type_info->flags_ & TF_IMMUTABLE) != 0)
@@ -393,8 +427,8 @@ class ArenaContainer {
bool allocContainer(container_size_t minSize); bool allocContainer(container_size_t minSize);
void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) { void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) {
obj->container_ = currentChunk_->asHeader();
obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo); obj->typeInfoOrMeta_ = const_cast<TypeInfo*>(typeInfo);
obj->setContainer(currentChunk_->asHeader());
// Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC. // Here we do not take into account typeInfo's immutability for ARC strategy, as there's no ARC.
} }
runtime/src/main/cpp/ObjCExport.mm
+1 -1
View File
@@ -113,7 +113,7 @@ static void initializeClass(Class clazz);
extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject); extern "C" ALWAYS_INLINE void Kotlin_ObjCExport_releaseAssociatedObject(void* associatedObject);
static inline id AtomicSetAssociatedObject(ObjHeader* obj, id associatedObject) { static inline id AtomicSetAssociatedObject(ObjHeader* obj, id associatedObject) {
if (!obj->container()->permanentOrFrozen()) { if (!PermanentOrFrozen(obj)) {
SetAssociatedObject(obj, associatedObject); SetAssociatedObject(obj, associatedObject);
return associatedObject; return associatedObject;
} else { } else {
runtime/src/main/cpp/Worker.cpp
+1 -1
View File
@@ -547,7 +547,7 @@ void Kotlin_Worker_freezeInternal(KRef object) {
} }
KBoolean Kotlin_Worker_isFrozenInternal(KRef object) { KBoolean Kotlin_Worker_isFrozenInternal(KRef object) {
return object == nullptr || object->container()->permanentOrFrozen(); return object == nullptr || PermanentOrFrozen(object);
} }
void Kotlin_Worker_ensureNeverFrozen(KRef object) { void Kotlin_Worker_ensureNeverFrozen(KRef object) {
shared/src/main/kotlin/org/jetbrains/kotlin/konan/KonanAbiVersion.kt
+1 -1
View File
@@ -6,7 +6,7 @@ fun String.parseKonanAbiVersion(): KonanAbiVersion {
data class KonanAbiVersion(val version: Int) { data class KonanAbiVersion(val version: Int) {
companion object { companion object {
val CURRENT = KonanAbiVersion(4) val CURRENT = KonanAbiVersion(5)
} }
override fun toString() = "$version" override fun toString() = "$version"
} }