From e84e76107fe3f5e5911a6a6e69ff299f1f13ebd9 Mon Sep 17 00:00:00 2001 From: Igor Chevdar Date: Thu, 29 Mar 2018 19:14:49 +0300 Subject: [PATCH] Supported cycles during object subgraph freezing --- .../jetbrains/kotlin/backend/konan/ir/Ir.kt | 2 +- .../kotlin/backend/konan/llvm/Runtime.kt | 1 + .../backend/konan/llvm/StaticObjects.kt | 8 +- runtime/src/main/cpp/Memory.cpp | 191 +++++++++++++----- runtime/src/main/cpp/Memory.h | 27 +-- .../src/main/kotlin/konan/worker/Freezing.kt | 10 - 6 files changed, 162 insertions(+), 77 deletions(-) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt index 2faed371105..e3b871e29ff 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/ir/Ir.kt @@ -259,7 +259,7 @@ internal class KonanSymbols(context: Context, val symbolTable: SymbolTable): Sym context.getInternalFunctions("initInstance").single()) val freeze = symbolTable.referenceSimpleFunction( - builtInsPackage("konan", "worker").getContributedFunctions(Name.identifier("freezeAllowCycles"), NoLookupLocation.FROM_BACKEND).single()) + builtInsPackage("konan", "worker").getContributedFunctions(Name.identifier("freeze"), NoLookupLocation.FROM_BACKEND).single()) val getContinuation = symbolTable.referenceSimpleFunction( context.getInternalFunctions("getContinuation").single()) diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt index c5013639891..695d9abd4d1 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/Runtime.kt @@ -42,6 +42,7 @@ class Runtime(bitcodeFile: String) { val objHeaderType = getStructType("ObjHeader") val objHeaderPtrType = pointerType(objHeaderType) val arrayHeaderType = getStructType("ArrayHeader") + val containerHeaderType = getStructType("ContainerHeader") val frameOverlayType = getStructType("FrameOverlay") diff --git a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt index 9b89e61da6e..78fec34b014 100644 --- a/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt +++ b/backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/StaticObjects.kt @@ -32,14 +32,14 @@ import org.jetbrains.kotlin.types.TypeProjection import org.jetbrains.kotlin.types.replace private fun StaticData.objHeader(typeInfo: ConstPointer): Struct { - val containerOffsetNegative = 0 // Static object mark. - return Struct(runtime.objHeaderType, typeInfo, Int32(containerOffsetNegative)) + val container = NullPointer(runtime.containerHeaderType) // Static object mark. + return Struct(runtime.objHeaderType, typeInfo, container) } private fun StaticData.arrayHeader(typeInfo: ConstPointer, length: Int): Struct { assert (length >= 0) - val containerOffsetNegative = 0 // Static object mark. - return Struct(runtime.arrayHeaderType, typeInfo, Int32(containerOffsetNegative), Int32(length)) + val container = NullPointer(runtime.containerHeaderType) // Static object mark. + return Struct(runtime.arrayHeaderType, typeInfo, container, Int32(length)) } internal fun StaticData.createKotlinStringLiteral(value: String): ConstPointer { diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 8982cd2e67c..3d914e8caeb 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -255,6 +255,7 @@ struct MemoryState { size_t gcThreshold; // If collection is in progress. bool gcInProgress; + int finalizerQueueSuspendCount; #if GC_ERGONOMICS uint64_t lastGcTimestamp; @@ -367,6 +368,10 @@ inline bool isArena(const ContainerHeader* header) { return header->stack(); } +inline bool isAggregatingFrozenContainer(const ContainerHeader* header) { + return header->frozen() && header->objectCount() > 1; +} + inline container_size_t alignUp(container_size_t size, int alignment) { return (size + alignment - 1) & ~(alignment - 1); } @@ -505,7 +510,8 @@ inline void processFinalizerQueue(MemoryState* state) { #if TRACE_MEMORY state->containers->erase(container); #endif - runDeallocationHooks(container); + if (!isAggregatingFrozenContainer(container)) + runDeallocationHooks(container); CONTAINER_DESTROY_EVENT(state, container) konanFreeMemory(container); @@ -527,7 +533,7 @@ inline void scheduleDestroyContainer( #if USE_GC state->finalizerQueue->push_front(container); // We cannot clean finalizer queue while in GC. - if (!state->gcInProgress && state->finalizerQueue->size() > 256) { + if (!state->gcInProgress && state->finalizerQueueSuspendCount == 0 && state->finalizerQueue->size() > 256) { processFinalizerQueue(state); } #else @@ -849,12 +855,52 @@ ContainerHeader* AllocContainer(size_t size) { return result; } +ContainerHeader* AllocAggregatingFrozenContainer(KStdVector& containers) { + auto componentSize = containers.size(); + auto superContainer = AllocContainer(sizeof(ContainerHeader) + sizeof(void*) * componentSize); + auto place = reinterpret_cast(superContainer + 1); + for (auto* container : containers) { + *place++ = container; + // Set link to the new container. + auto obj = reinterpret_cast(container + 1); + obj->container_ = superContainer; + MEMORY_LOG("Set fictitious frozen container for %p: %p\n", obj, superContainer); + } + superContainer->setObjectCount(componentSize); + superContainer->freeze(); + return superContainer; +} + +void FreeAggregatingFrozenContainer(ContainerHeader* container) { + auto state = memoryState; + RuntimeAssert(isAggregatingFrozenContainer(container), "expected fictitious frozen container"); + MEMORY_LOG("%p is fictitious frozen container\n", container); + RuntimeAssert(!container->buffered(), "frozen objects must not participate in GC") + // Forbid finalizerQueue handling. + ++state->finalizerQueueSuspendCount; + // Special container for frozen objects. + ContainerHeader** subContainer = reinterpret_cast(container + 1); + MEMORY_LOG("Total subcontainers = %d\n", container->objectCount()); + for (int i = 0; i < container->objectCount(); ++i) { + MEMORY_LOG("Freeing subcontainer %p\n", *subContainer); + FreeContainer(*subContainer++); + } + --state->finalizerQueueSuspendCount; + scheduleDestroyContainer(state, container, false); + MEMORY_LOG("Freeing subcontainers done\n"); +} + void FreeContainer(ContainerHeader* header) { RuntimeAssert(!header->permanent(), "this kind of container shalln't be freed"); auto state = memoryState; CONTAINER_FREE_EVENT(state, header) + if (isAggregatingFrozenContainer(header)) { + FreeAggregatingFrozenContainer(header); + return; + } + // Now let's clean all object's fields in this container. traverseContainerObjectFields(header, [](ObjHeader** location) { UpdateRef(location, nullptr); @@ -1024,9 +1070,9 @@ MemoryState* InitMemory() { == offsetof(ObjHeader, typeInfoOrMeta_), "Layout mismatch"); - RuntimeAssert(offsetof(ArrayHeader, containerOffsetNegative_) + RuntimeAssert(offsetof(ArrayHeader, container_) == - offsetof(ObjHeader , containerOffsetNegative_), + offsetof(ObjHeader , container_), "Layout mismatch"); RuntimeAssert(offsetof(TypeInfo, typeInfo_) == @@ -1150,17 +1196,16 @@ OBJ_GETTER(InitSharedInstance, UpdateRef(localLocation, object); #if KONAN_NO_EXCEPTIONS ctor(object); - // TODO: uncomment as soon as cycles are correctly handled during freezing. - //if (!object->container()->frozen()) - //ThrowFreezingException(); + if (!object->container()->frozen()) + ThrowFreezingException(); UpdateRef(location, object); __sync_synchronize(); return object; #else try { ctor(object); - //if (!object->container()->frozen()) - //ThrowFreezingException(); + if (!object->container()->frozen()) + ThrowFreezingException(); UpdateRef(location, object); __sync_synchronize(); return object; @@ -1449,27 +1494,39 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) { * - not 'marked' and not 'seen' as WHITE marker (object is unprocessed) * When we see GREY during DFS, it means we see cycle. */ -void depthFirstTraversal(ContainerHeader* container, bool* hasCycles) { +void depthFirstTraversal(ContainerHeader* container, bool* hasCycles, KStdVector& order) { // Mark GRAY. container->setSeen(); - traverseContainerObjectFields(container, [&hasCycles](ObjHeader** location) { - ObjHeader* obj = *location; - if (obj != nullptr) { - ContainerHeader* objContainer = obj->container(); - if (!objContainer->permanentOrFrozen()) { - // Marked GREY, there's cycle. - if (objContainer->seen()) *hasCycles = true; - // Go deeper if WHITE. - if (!objContainer->seen() && !objContainer->marked()) { - depthFirstTraversal(objContainer, hasCycles); - } - } + traverseContainerReferredObjects(container, [hasCycles, &order](ObjHeader* obj) { + ContainerHeader* objContainer = obj->container(); + if (!objContainer->permanentOrFrozen()) { + // Marked GREY, there's cycle. + if (objContainer->seen()) *hasCycles = true; + + // Go deeper if WHITE. + if (!objContainer->seen() && !objContainer->marked()) { + depthFirstTraversal(objContainer, hasCycles, order); } - }); + } + }); // Mark BLACK. container->resetSeen(); container->mark(); + order.push_back(container); +} + +void traverseStronglyConnectedComponent(ContainerHeader* container, + KStdUnorderedMap> const& reversedEdges, + KStdVector& component) { + component.push_back(container); + container->mark(); + auto it = reversedEdges.find(container); + RuntimeAssert(it != reversedEdges.end(), "unknown node during condensation building"); + for (auto* nextContainer : it->second) { + if (!nextContainer->marked()) + traverseStronglyConnectedComponent(nextContainer, reversedEdges, component); + } } /** @@ -1503,47 +1560,91 @@ void FreezeSubgraph(ObjHeader* root) { // Do DFS cycle detection. bool hasCycles = false; - depthFirstTraversal(rootContainer, &hasCycles); + KStdVector order; + depthFirstTraversal(rootContainer, &hasCycles, order); + KStdUnorderedMap> reversedEdges; // Now unmark all marked objects, and freeze them, if no cycles detected. - KStdDeque stack; - stack.push_back(rootContainer); - while (!stack.empty()) { - ContainerHeader* current = stack.front(); - stack.pop_front(); + KStdDeque queue; + queue.push_back(rootContainer); + while (!queue.empty()) { + ContainerHeader* current = queue.front(); + queue.pop_front(); current->unMark(); - current->resetSeen(); - if (!hasCycles) { + if (hasCycles) { + reversedEdges.emplace(current, KStdVector(0)); + } else { current->resetBuffered(); current->setColor(CONTAINER_TAG_GC_BLACK); // Note, that once object is frozen, it could be concurrently accessed, so // color and similar attributes shall not be used. current->freeze(); } - traverseContainerObjectFields(current, [&hasCycles, &stack](ObjHeader** location) { - ObjHeader* obj = *location; - if (obj != nullptr) { - ContainerHeader* objContainer = obj->container(); - if (!objContainer->permanentOrFrozen() && objContainer->marked()) - stack.push_back(objContainer); + traverseContainerReferredObjects(current, [hasCycles, current, &queue, &reversedEdges](ObjHeader* obj) { + ContainerHeader* objContainer = obj->container(); + if (!objContainer->permanentOrFrozen()) { + if (objContainer->marked()) + queue.push_back(objContainer); + if (hasCycles) + reversedEdges.emplace(objContainer, KStdVector(0)).first->second.push_back(current); } }); } + if (hasCycles) { + KStdVector> components; + MEMORY_LOG("Condensation:\n"); + // Enumerate in topological order. + for (auto it = order.rbegin(); it != order.rend(); ++it) { + auto* container = *it; + if (container->marked()) continue; + KStdVector component; + traverseStronglyConnectedComponent(container, reversedEdges, component); + MEMORY_LOG("SCC:\n"); +#if TRACE_MEMORY + for (auto c : component) + konan::consolePrintf(" %p\n", c); +#endif + components.push_back(std::move(component)); + } + // Enumerate strongly connected components in reversed topological order. + for (auto it = components.rbegin(); it != components.rend(); ++it) { + auto& component = *it; + int internalRefsCount = 0; + int totalCount = 0; + for (auto* container : component) { + totalCount += container->refCount(); + traverseContainerReferredObjects(container, [&internalRefsCount](ObjHeader* obj) { + if (!obj->container()->permanentOrFrozen()) + ++internalRefsCount; + }); + } + auto superContainer = component.size() == 1 + ? component[0] + : AllocAggregatingFrozenContainer(component); // Create fictitious container for the whole component. + // Don't count internal references. + superContainer->setRefCount(totalCount - internalRefsCount); + + // Freeze component. + for (auto* container : component) { + container->resetBuffered(); + container->setColor(CONTAINER_TAG_GC_BLACK); + // Note, that once object is frozen, it could be concurrently accessed, so + // color and similar attributes shall not be used. + container->freeze(); + } + } + } + // Now remove frozen objects from the toFree list. // TODO: optimize it by keeping ignored (i.e. freshly frozen) objects in the set, // and use it when analyzing toFree during collection. auto state = memoryState; - for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { - auto container = *it; - if (container->frozen()) { - *it = markAsRemoved(container); - } + for (auto& container : *(state->toFree)) { + if (!isMarkedAsRemoved(container) && container->frozen()) + container = markAsRemoved(container); } - - // For now, just throw an exception here. - if (hasCycles) ThrowFreezingException(); } // This function is called from field mutators to check if object's header is frozen. diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 685f0cf8429..5e84fe1b70c 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -57,7 +57,6 @@ typedef enum { CONTAINER_TAG_GC_SEEN = 1 << 4 } ContainerTag; -typedef uint32_t container_offset_t; typedef uint32_t container_size_t; // Header of all container objects. Contains reference counter. @@ -92,6 +91,10 @@ struct ContainerHeader { return refCount_ >> CONTAINER_TAG_SHIFT; } + inline void setRefCount(unsigned refCount) { + refCount_ = tag() | (refCount << CONTAINER_TAG_SHIFT); + } + template inline void incRefCount() { #ifdef KONAN_NO_THREADS @@ -182,7 +185,7 @@ struct MetaObjHeader; // Header of every object. struct ObjHeader { TypeInfo* typeInfoOrMeta_; - container_offset_t containerOffsetNegative_; + ContainerHeader* container_; const TypeInfo* type_info() const { return typeInfoOrMeta_->typeInfo_; @@ -200,12 +203,7 @@ struct ObjHeader { static ContainerHeader theStaticObjectsContainer; ContainerHeader* container() const { - if (containerOffsetNegative_ == 0) { - return &theStaticObjectsContainer; - } else { - return reinterpret_cast( - reinterpret_cast(this) - containerOffsetNegative_); - } + return container_ == nullptr ? &theStaticObjectsContainer : container_; } // Unsafe cast to ArrayHeader. Use carefully! @@ -223,15 +221,14 @@ struct ObjHeader { // Header of value type array objects. Keep layout in sync with that of object header. struct ArrayHeader { TypeInfo* typeInfoOrMeta_; - container_offset_t containerOffsetNegative_; + ContainerHeader* container_; const TypeInfo* type_info() const { return typeInfoOrMeta_->typeInfo_; } ContainerHeader* container() const { - return reinterpret_cast( - reinterpret_cast(this) - containerOffsetNegative_); + return container_; } ObjHeader* obj() { return reinterpret_cast(this); } @@ -259,10 +256,8 @@ class Container { ContainerHeader* header_; void SetHeader(ObjHeader* obj, const TypeInfo* type_info) { - obj->containerOffsetNegative_ = - reinterpret_cast(obj) - reinterpret_cast(header_); + obj->container_ = header_; obj->typeInfoOrMeta_ = const_cast(type_info); - RuntimeAssert(obj->container() == header_, "Placement must match"); } }; @@ -338,10 +333,8 @@ class ArenaContainer { bool allocContainer(container_size_t minSize); void setHeader(ObjHeader* obj, const TypeInfo* typeInfo) { - obj->containerOffsetNegative_ = - reinterpret_cast(obj) - reinterpret_cast(currentChunk_->asHeader()); + obj->container_ = currentChunk_->asHeader(); obj->typeInfoOrMeta_ = const_cast(typeInfo); - RuntimeAssert(obj->container() == currentChunk_->asHeader(), "Placement must match"); } ContainerChunk* currentChunk_; diff --git a/runtime/src/main/kotlin/konan/worker/Freezing.kt b/runtime/src/main/kotlin/konan/worker/Freezing.kt index 49925a2f318..40e14cd90d4 100644 --- a/runtime/src/main/kotlin/konan/worker/Freezing.kt +++ b/runtime/src/main/kotlin/konan/worker/Freezing.kt @@ -37,16 +37,6 @@ fun T.freeze(): T { return this } -// TODO: Remove. -fun T.freezeAllowCycles(): T { - try { - freezeInternal(this) - } catch (t: FreezingException) { - return this - } - return this -} - val Any?.isFrozen get() = isFrozenInternal(this)