diff --git a/runtime/src/main/cpp/Memory.cpp b/runtime/src/main/cpp/Memory.cpp index 995f9406304..34d20670928 100644 --- a/runtime/src/main/cpp/Memory.cpp +++ b/runtime/src/main/cpp/Memory.cpp @@ -26,10 +26,8 @@ #include "Natives.h" // If garbage collection algorithm for cyclic garbage to be used. +// We are using the Bacon's algorithm for GC (http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon03Pure.pdf). #define USE_GC 1 -// Optimize management of cyclic garbage (increases memory footprint). -// Not recommended for low-end embedded targets. -#define OPTIMIZE_GC 1 // Define to 1 to print all memory operations. #define TRACE_MEMORY 0 // Trace garbage collection phases. @@ -81,23 +79,27 @@ struct MemoryState { // Finalizer queue. ContainerHeaderDeque* finalizerQueue; - // Set of references to release. - ContainerHeaderSet* toFree; + /* + * Typical scenario for GC is as following: + * we have 90% of objects with refcount = 0 which will be deleted during + * the first phase of the algorithm. + * We could mark them with a bit in order to tell the next two phases to skip them + * and thus requiring only one list, but the downside is that both of the + * next phases would iterate over the whole list of objects instead of only 10%. + */ + ContainerHeaderList* toFree; // List of all cycle candidates. + ContainerHeaderList* roots; // Real candidates excluding those with refcount = 0. // How many GC suspend requests happened. int gcSuspendCount; // How many candidate elements in toFree shall trigger collection. size_t gcThreshold; // If collection is in progress. bool gcInProgress; -#if OPTIMIZE_GC - // Cache backed by toFree set. - ContainerHeader** toFreeCache; - // Current number of elements in the cache. - uint32_t cacheSize; -#endif -#endif +#endif // USE_GC }; +void FreeContainer(ContainerHeader* header); + namespace { // TODO: can we pass this variable as an explicit argument? @@ -113,6 +115,10 @@ inline bool isPermanent(const ContainerHeader* header) { return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT; } +inline bool isArena(const ContainerHeader* header) { + return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK; +} + inline container_size_t alignUp(container_size_t size, int alignment) { return (size + alignment - 1) & ~(alignment - 1); } @@ -145,14 +151,62 @@ inline bool isRefCounted(KConstRef object) { return (object->container()->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_NORMAL; } +} // namespace + +extern "C" { +void objc_release(void* ptr); +} + +inline void runDeallocationHooks(ObjHeader* obj) { +#if KONAN_OBJC_INTEROP + if (obj->type_info() == theObjCPointerHolderTypeInfo) { + void* objcPtr = *reinterpret_cast(obj + 1); // TODO: use more reliable layout description + objc_release(objcPtr); + } +#endif +} + +inline void runDeallocationHooks(ContainerHeader* container) { + ObjHeader* obj = reinterpret_cast(container + 1); + + for (int index = 0; index < container->objectCount(); index++) { + runDeallocationHooks(obj); + + obj = reinterpret_cast( + reinterpret_cast(obj) + objectSize(obj)); + } +} + +static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) { + for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { + ObjHeader** location = reinterpret_cast( + reinterpret_cast(body) + typeInfo->objOffsets_[index]); +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from DeinitInstanceBodyImpl\n"); +#endif + UpdateRef(location, nullptr); + } +} + +void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) { + DeinitInstanceBodyImpl(typeInfo, body); +} + +namespace { #if USE_GC - inline void processFinalizerQueue(MemoryState* state) { // TODO: reuse elements of finalizer queue for new allocations. while (!state->finalizerQueue->empty()) { auto container = memoryState->finalizerQueue->back(); state->finalizerQueue->pop_back(); + if ((reinterpret_cast(container) & 1) != 0) { + container = reinterpret_cast(reinterpret_cast(container) & ~1); +#if TRACE_MEMORY + state->containers->erase(container); +#endif + runDeallocationHooks(container); + } konanFreeMemory(container); state->allocCount--; } @@ -174,135 +228,95 @@ inline void scheduleDestroyContainer( } -#if USE_GC +#if !USE_GC -inline uint32_t hashOf(ContainerHeader* container) { - uintptr_t value = reinterpret_cast(container); - return static_cast(value >> 3) ^ static_cast(static_cast(value) >> 32); +inline void IncrementRC(ContainerHeader* container) { + container->incRefCount(); } +inline void DecrementRC(ContainerHeader* container) { + if (container->decRefCount() == 0) { + FreeContainer(container); + } +} + +#else // USE_GC + inline uint32_t freeableSize(MemoryState* state) { -#if OPTIMIZE_GC - return state->cacheSize + state->toFree->size(); -#else return state->toFree->size(); -#endif } -inline void addFreeable(MemoryState* state, ContainerHeader* container) { - if (memoryState->toFree == nullptr || !isFreeable(container)) - return; -#if OPTIMIZE_GC - auto hash = hashOf(container) % state->gcThreshold; - auto value = state->toFreeCache[hash]; - if (value == container) { - return; - } - if (value == nullptr) { - memoryState->cacheSize++; - state->toFreeCache[hash] = container; - return; - } - state->toFree->insert(container); - if (value != (ContainerHeader*)0x1) { - memoryState->cacheSize--; - state->toFree->insert(value); - state->toFreeCache[hash] = (ContainerHeader*)0x1; - } -#else - state->toFree->insert(container); -#endif - if (state->gcSuspendCount == 0 && - freeableSize(memoryState) > state->gcThreshold) { - GarbageCollect(); - } +inline void IncrementRC(ContainerHeader* container) { + container->incRefCount(); + container->setColor(CONTAINER_TAG_GC_BLACK); } -inline void removeFreeable(MemoryState* state, ContainerHeader* container) { - if (state->toFree == nullptr || !isFreeable(container)) - return; -#if OPTIMIZE_GC - auto hash = hashOf(container) % state->gcThreshold; - auto value = state->toFreeCache[hash]; - if (value == container) { - state->cacheSize--; - state->toFreeCache[hash] = nullptr; - return; - } -#endif - state->toFree->erase(container); -} - -// Must only be called in context of GC. -inline void flushFreeableCache(MemoryState* state) { -#if OPTIMIZE_GC - for (auto i = 0; i < state->gcThreshold; i++) { - if ((uintptr_t)state->toFreeCache[i] > 0x1) { - state->toFree->insert(state->toFreeCache[i]); +inline void DecrementRC(ContainerHeader* container) { + if (container->decRefCount() == 0) { + FreeContainer(container); + } else { // Possible root. + if (container->color() != CONTAINER_TAG_GC_PURPLE) { + container->setColor(CONTAINER_TAG_GC_PURPLE); + if (!container->buffered()) { + container->setBuffered(); + auto state = memoryState; + state->toFree->push_back(container); + if (state->gcSuspendCount == 0 && freeableSize(state) > state->gcThreshold) { + GarbageCollect(); + } + } } } - // Mass-clear cache. - memset(state->toFreeCache, 0, - sizeof(ContainerHeader*) * state->gcThreshold); - state->cacheSize = 0; -#endif } inline void initThreshold(MemoryState* state, uint32_t gcThreshold) { -#if OPTIMIZE_GC - if (state->toFreeCache != nullptr) { - GarbageCollect(); - konanFreeMemory(state->toFreeCache); - } - state->toFreeCache = reinterpret_cast( - konanAllocMemory(sizeof(ContainerHeader*) * gcThreshold)); - state->cacheSize = 0; -#endif state->gcThreshold = gcThreshold; } +#endif // USE_GC -// Must be vector or map 'container -> number', to keep reference counters correct. -ContainerHeaderList collectMutableReferred(ContainerHeader* header) { - ContainerHeaderList result; - ObjHeader* obj = reinterpret_cast(header + 1); - for (int object = 0; object < header->objectCount(); object++) { +template +void traverseContainerObjectFields(ContainerHeader* container, func process) { + ObjHeader* obj = reinterpret_cast(container + 1); + for (int object = 0; object < container->objectCount(); object++) { const TypeInfo* typeInfo = obj->type_info(); - // TODO: generalize iteration over all references. for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { ObjHeader** location = reinterpret_cast( reinterpret_cast(obj + 1) + typeInfo->objOffsets_[index]); - ObjHeader* ref = *location; - if (ref != nullptr && !isPermanent(ref->container())) { - result.push_back(ref->container()); - } + process(location); } if (typeInfo == theArrayTypeInfo) { ArrayHeader* array = obj->array(); for (int index = 0; index < array->count_; index++) { - ObjHeader* ref = *ArrayAddressOfElementAt(array, index); - if (ref != nullptr && !isPermanent(ref->container())) { - result.push_back(ref->container()); - } + process(ArrayAddressOfElementAt(array, index)); } } obj = reinterpret_cast( reinterpret_cast(obj) + objectSize(obj)); } - return result; } +template +void traverseContainerReferredObjects(ContainerHeader* container, func process) { + traverseContainerObjectFields(container, [process](ObjHeader** location) { + ObjHeader* ref = *location; + if (ref != nullptr) process(ref); + }); +} + +#if TRACE_MEMORY || USE_GC + void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) { - fprintf(stderr, "%s: %p (%08x): %d refs %s\n", + fprintf(stderr, "%s: %p (%08x): %d refs\n", prefix, - header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT, - (header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-"); + header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT); seen->insert(header); - auto children = collectMutableReferred(header); - for (auto child : children) { - if (seen->count(child) == 0) { + traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) { + auto child = ref->container(); + RuntimeAssert(!isArena(child), "A reference to local object is encountered"); + if (!isPermanent(child) && (seen->count(child) == 0)) { dumpWorker(prefix, child, seen); } - } + }); } void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { @@ -313,71 +327,159 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { } } -void phase1(ContainerHeader* header) { - if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) - return; - header->refCount_ |= CONTAINER_TAG_SEEN; - auto containers = collectMutableReferred(header); - for (auto container : containers) { - container->decRefCount(); - phase1(container); +#endif + +void MarkRoots(MemoryState*); +void DeleteCorpses(MemoryState*); +void ScanRoots(MemoryState*); +void CollectRoots(MemoryState*); +void MarkGray(ContainerHeader* container); +void Scan(ContainerHeader* container); +void ScanBlack(ContainerHeader* container); +void CollectWhite(MemoryState*, ContainerHeader* container); + +void CollectCycles(MemoryState* state) { + MarkRoots(state); + ScanRoots(state); + CollectRoots(state); + state->toFree->clear(); + state->roots->clear(); +} + +void MarkRoots(MemoryState* state) { + for (auto container : *(state->toFree)) { + if ((reinterpret_cast(container) & 1) != 0) + continue; + auto color = container->color(); + auto rcIsZero = container->refCount() == 0; + if (color == CONTAINER_TAG_GC_PURPLE && !rcIsZero) { + MarkGray(container); + state->roots->push_back(container); + } else { + container->resetBuffered(); + if (color == CONTAINER_TAG_GC_BLACK && rcIsZero) { + scheduleDestroyContainer(state, reinterpret_cast(reinterpret_cast(container) | 1)); + } + } } } -void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) { - if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0) - return; - if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0) - rootset->insert(header); - header->refCount_ &= ~CONTAINER_TAG_SEEN; - auto containers = collectMutableReferred(header); - for (auto container : containers) { - phase2(container, rootset); +void ScanRoots(MemoryState* state) { + for (auto container : *(state->roots)) { + Scan(container); } } -void phase3(ContainerHeader* header) { - if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) { +void CollectRoots(MemoryState* state) { + for (auto container : *(state->roots)) { + container->resetBuffered(); + CollectWhite(state, container); + } +} + +void MarkGray(ContainerHeader* container) { + if (container->color() == CONTAINER_TAG_GC_GRAY) return; + container->setColor(CONTAINER_TAG_GC_GRAY); + traverseContainerReferredObjects(container, [](ObjHeader* ref) { + auto childContainer = ref->container(); + RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); + if (!isPermanent(childContainer)) { + childContainer->decRefCount(); + MarkGray(childContainer); + } + }); +} + +void Scan(ContainerHeader* container) { + if (container->color() != CONTAINER_TAG_GC_GRAY) return; + if (container->refCount() != 0) { + ScanBlack(container); return; } - header->refCount_ |= CONTAINER_TAG_SEEN; - auto containers = collectMutableReferred(header); - for (auto container : containers) { - container->incRefCount(); - phase3(container); + container->setColor(CONTAINER_TAG_GC_WHITE); + traverseContainerReferredObjects(container, [](ObjHeader* ref) { + auto childContainer = ref->container(); + RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); + if (!isPermanent(childContainer)) { + Scan(childContainer); + } + }); +} + +void ScanBlack(ContainerHeader* container) { + container->setColor(CONTAINER_TAG_GC_BLACK); + traverseContainerReferredObjects(container, [](ObjHeader* ref) { + auto childContainer = ref->container(); + RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); + if (!isPermanent(childContainer)) { + childContainer->incRefCount(); + if (childContainer->color() != CONTAINER_TAG_GC_BLACK) + ScanBlack(childContainer); + } + }); +} + +void CollectWhite(MemoryState* state, ContainerHeader* container) { + if (container->color() != CONTAINER_TAG_GC_WHITE + || container->buffered()) + return; + container->setColor(CONTAINER_TAG_GC_BLACK); + traverseContainerReferredObjects(container, [state](ObjHeader* ref) { + auto childContainer = ref->container(); + RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered"); + if (!isPermanent(childContainer)) { + CollectWhite(state, childContainer); + } + }); + scheduleDestroyContainer(state, reinterpret_cast(reinterpret_cast(container) | 1)); +} + +inline void AddRef(ContainerHeader* header) { + // Looking at container type we may want to skip AddRef() totally + // (non-escaping stack objects, constant objects). + switch (header->refCount_ & CONTAINER_TAG_MASK) { + case CONTAINER_TAG_STACK: + case CONTAINER_TAG_PERMANENT: + break; + case CONTAINER_TAG_NORMAL: + IncrementRC(header); + break; + default: + RuntimeAssert(false, "unknown container type"); + break; } } -void phase4(MemoryState* state, ContainerHeader* header) { - auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT; - bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) || - (refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0); - if (seen) return; - - // Add to finalize queue and update seen bit. - if (refCount == 0) { - scheduleDestroyContainer(state, header); - header->refCount_ |= CONTAINER_TAG_SEEN; - } else { - header->refCount_ &= ~CONTAINER_TAG_SEEN; - } - auto containers = collectMutableReferred(header); - for (auto container : containers) { - phase4(state, container); +inline void Release(ContainerHeader* header) { + // Looking at container type we may want to skip Release() totally + // (non-escaping stack objects, constant objects). + switch (header->refCount_ & CONTAINER_TAG_MASK) { + case CONTAINER_TAG_PERMANENT: + case CONTAINER_TAG_STACK: + break; + case CONTAINER_TAG_NORMAL: + DecrementRC(header); + break; + default: + RuntimeAssert(false, "unknown container type"); + break; } } -#endif // USE_GC - // We use first slot as place to store frame-local arena container. // TODO: create ArenaContainer object on the stack, so that we don't // do two allocations per frame (ArenaContainer + actual container). inline ArenaContainer* initedArena(ObjHeader** auxSlot) { - ObjHeader* slotValue = *auxSlot; - if (slotValue) return reinterpret_cast(slotValue); - ArenaContainer* arena = konanConstructInstance(); - arena->Init(); - *auxSlot = reinterpret_cast(arena); + auto frame = asFrameOverlay(auxSlot); +#if TRACE_MEMORY + fprintf(stderr, "Initializing arena at %p\n", frame); +#endif + auto arena = frame->arena; + if (!arena) { + arena = konanConstructInstance(); + arena->Init(); + frame->arena = arena; + } return arena; } @@ -398,90 +500,41 @@ ContainerHeader* AllocContainer(size_t size) { return result; } -extern "C" { -void objc_release(void* ptr); -} - -inline void runDeallocationHooks(ObjHeader* obj) { -#if KONAN_OBJC_INTEROP - if (obj->type_info() == theObjCPointerHolderTypeInfo) { - void* objcPtr = *reinterpret_cast(obj + 1); // TODO: use more reliable layout description - objc_release(objcPtr); - } -#endif -} - -static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) { - for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { - ObjHeader** location = reinterpret_cast( - reinterpret_cast(body) + typeInfo->objOffsets_[index]); - UpdateRef(location, nullptr); - } -} - -void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) { - DeinitInstanceBodyImpl(typeInfo, body); -} - void FreeContainer(ContainerHeader* header) { RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed"); auto state = memoryState; #if TRACE_MEMORY if (isFreeable(header)) { - fprintf(stderr, "<<< free %p\n", header); - state->containers->erase(header); + fprintf(stderr, "<<< free %p\n", header); } #endif -#if USE_GC - removeFreeable(state, header); -#endif // Now let's clean all object's fields in this container. - ObjHeader* obj = reinterpret_cast(header + 1); + traverseContainerObjectFields(header, [](ObjHeader** location) { +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from FreeContainer\n"); +#endif - for (int index = 0; index < header->objectCount(); index++) { - runDeallocationHooks(obj); - - const TypeInfo* typeInfo = obj->type_info(); - - DeinitInstanceBodyImpl(typeInfo, reinterpret_cast(obj + 1)); - - // Object arrays are *special*. - if (typeInfo == theArrayTypeInfo) { - ArrayHeader* array = obj->array(); - ReleaseRefs(ArrayAddressOfElementAt(array, 0), array->count_); - } - obj = reinterpret_cast( - reinterpret_cast(obj) + objectSize(obj)); - } + UpdateRef(location, nullptr); + }); // And release underlying memory. - if (isFreeable(header)) { - scheduleDestroyContainer(state, header); - } -} + if (!isFreeable(header)) { + runDeallocationHooks(header); + } else { + header->setColor(CONTAINER_TAG_GC_BLACK); + if (!header->buffered()) { + + runDeallocationHooks(header); -#if USE_GC -void FreeContainerNoRef(MemoryState* state, ContainerHeader* header) { - RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed"); #if TRACE_MEMORY - fprintf(stderr, "<<< free %p\n", header); - state->containers->erase(header); + memoryState->containers->erase(header); #endif -#if USE_GC - removeFreeable(state, header); -#endif - ObjHeader* obj = reinterpret_cast(header + 1); - for (int index = 0; index < header->objectCount(); index++) { - runDeallocationHooks(obj); - obj = reinterpret_cast( - reinterpret_cast(obj) + objectSize(obj)); + scheduleDestroyContainer(state, header); + } } - - scheduleDestroyContainer(state, header); } -#endif void ObjectContainer::Init(const TypeInfo* type_info) { RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object"); @@ -525,9 +578,15 @@ void ArenaContainer::Init() { } void ArenaContainer::Deinit() { +#if TRACE_MEMORY + fprintf(stderr, "Arena::Deinit start\n"); +#endif auto chunk = currentChunk_; while (chunk != nullptr) { // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set. +#if TRACE_MEMORY + fprintf(stderr, "Arena::Deinit free chunk\n"); +#endif FreeContainer(chunk->asHeader()); chunk = chunk->next; } @@ -537,7 +596,9 @@ void ArenaContainer::Deinit() { chunk = chunk->next; konanFreeMemory(toRemove); } - +#if TRACE_MEMORY + fprintf(stderr, "Arena::Deinit end\n"); +#endif } bool ArenaContainer::allocContainer(container_size_t minSize) { @@ -588,7 +649,7 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) { uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader); ObjHeader* result = reinterpret_cast(place(size)); if (!result) { - return nullptr; + return nullptr; } currentChunk_->asHeader()->incObjectCount(); setMeta(result, type_info); @@ -613,29 +674,13 @@ inline void AddRef(const ObjHeader* object) { fprintf(stderr, "AddRef on %p in %p\n", object, object->container()); #endif AddRef(object->container()); -#if USE_GC - // TODO: one could remove from toFree set here, as now container is reachable - // from the rootset, so cannot be cycle collection candidate. - // removeFreeable(memoryState, object->container()); -#endif } inline void ReleaseRef(const ObjHeader* object) { #if TRACE_MEMORY fprintf(stderr, "ReleaseRef on %p in %p\n", object, object->container()); #endif -#if USE_GC - // If object is not a cycle candidate - just return. - if (Release(object->container())) { - return; - } -#if TRACE_MEMORY - fprintf(stderr, "%p is release candidate\n", object->container()); -#endif - addFreeable(memoryState, object->container()); -#else // !USE_GC Release(object->container()); -#endif // USE_GC } extern "C" { @@ -649,6 +694,7 @@ MemoryState* InitMemory() { == offsetof(ObjHeader , container_offset_negative_), "Layout mismatch"); + RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers") RuntimeAssert(memoryState == nullptr, "memory state must be clear"); memoryState = konanConstructInstance(); // TODO: initialize heap here. @@ -659,7 +705,8 @@ MemoryState* InitMemory() { #endif #if USE_GC memoryState->finalizerQueue = konanConstructInstance(); - memoryState->toFree = konanConstructInstance(); + memoryState->toFree = konanConstructInstance(); + memoryState->roots = konanConstructInstance(); memoryState->gcInProgress = false; initThreshold(memoryState, kGcThreshold); memoryState->gcSuspendCount = 0; @@ -672,6 +719,9 @@ void DeinitMemory(MemoryState* memoryState) { // Free all global objects, to ensure no memory leaks happens. for (auto location: *memoryState->globalObjects) { fprintf(stderr, "Release global in *%p: %p\n", location, *location); +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from DeinitMemory\n"); +#endif UpdateRef(location, nullptr); } konanDestructInstance(memoryState->globalObjects); @@ -680,30 +730,26 @@ void DeinitMemory(MemoryState* memoryState) { #if USE_GC GarbageCollect(); + RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC"); konanDestructInstance(memoryState->toFree); - memoryState->toFree = nullptr; - -#if OPTIMIZE_GC - if (memoryState->toFreeCache != nullptr) { - konanFreeMemory(memoryState->toFreeCache); - memoryState->toFreeCache = nullptr; - } -#endif + konanDestructInstance(memoryState->roots); konanDestructInstance(memoryState->finalizerQueue); memoryState->finalizerQueue = nullptr; #endif // USE_GC +#if TRACE_MEMORY if (memoryState->allocCount > 0) { fprintf(stderr, "*** Memory leaks, leaked %d containers ***\n", memoryState->allocCount); -#if TRACE_MEMORY dumpReachable("", memoryState->containers); - konanDestructInstance(memoryState->containers); - memoryState->containers = nullptr; -#endif } + konanDestructInstance(memoryState->containers); + memoryState->containers = nullptr; +#else + RuntimeAssert(memoryState->allocCount == 0, "Memory leaks found"); +#endif konanFreeMemory(memoryState); ::memoryState = nullptr; @@ -745,6 +791,9 @@ OBJ_GETTER(InitInstance, } ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from InitInstance\n"); +#endif UpdateRef(location, object); #if KONAN_NO_EXCEPTIONS ctor(object); @@ -760,7 +809,13 @@ OBJ_GETTER(InitInstance, #endif return object; } catch (...) { +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from InitInstance #2\n"); +#endif UpdateRef(OBJ_RESULT, nullptr); +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from InitInstance #3\n"); +#endif UpdateRef(location, nullptr); throw; } @@ -772,9 +827,7 @@ void SetRef(ObjHeader** location, const ObjHeader* object) { fprintf(stderr, "SetRef *%p: %p\n", location, object); #endif *const_cast(location) = object; - if (object != nullptr) { - AddRef(object); - } + AddRef(object); } ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { @@ -797,16 +850,20 @@ void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) { auto arena = initedArena(asArenaSlot(returnSlot)); returnSlot = arena->getSlot(); } +#if TRACE_MEMORY + fprintf(stderr, "Calling UpdateRef from UpdateReturnRef\n"); +#endif UpdateRef(returnSlot, object); } void UpdateRef(ObjHeader** location, const ObjHeader* object) { RuntimeAssert(!isArenaSlot(location), "must not be a slot"); ObjHeader* old = *location; + if (old != object) { #if TRACE_MEMORY fprintf(stderr, "UpdateRef *%p: %p -> %p\n", location, old, object); + fprintf(stderr, " *%p: %p -> %p\n", location, old == nullptr ? nullptr : old->container(), object == nullptr ? nullptr : object->container()); #endif - if (old != object) { if (object != nullptr) { AddRef(object); } @@ -835,6 +892,9 @@ void LeaveFrame(ObjHeader** start, int count) { #endif arena->Deinit(); konanFreeMemory(arena); +#if TRACE_MEMORY + fprintf(stderr, "LeaveFrame: free arena done %p\n", arena); +#endif } } @@ -843,6 +903,7 @@ void ReleaseRefs(ObjHeader** start, int count) { fprintf(stderr, "ReleaseRefs %p .. %p\n", start, start + count); #endif ObjHeader** current = start; + auto state = memoryState; while (count-- > 0) { ObjHeader* object = *current; if (object != nullptr) { @@ -855,64 +916,21 @@ void ReleaseRefs(ObjHeader** start, int count) { } #if USE_GC + void GarbageCollect() { MemoryState* state = memoryState; - RuntimeAssert(state->toFree != nullptr, "GC must not be stopped"); RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed"); +#if TRACE_MEMORY + fprintf(stderr, "Garbage collect\n"); +#endif + state->gcInProgress = true; - // Flush cache. - flushFreeableCache(state); - - // Traverse inner pointers in the closure of release candidates, and - // temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing. -#if TRACE_GC_PHASES - dumpReachable("P0", state->toFree); -#endif - for (auto container : *state->toFree) { - phase1(container); + while (state->toFree->size() > 0) { + CollectCycles(state); + processFinalizerQueue(state); } -#if TRACE_GC_PHASES - dumpReachable("P1", state->toFree); -#endif - - // Collect rootset from containers with non-zero reference counter. Those must - // be referenced from outside of newly released object graph. - // Clear CONTAINER_TAG_SEEN while traversing. - ContainerHeaderSet rootset; - for (auto container : *state->toFree) { - phase2(container, &rootset); - } -#if TRACE_GC_PHASES - dumpReachable("P2", state->toFree); -#endif - - // Increment references for all elements reachable from the rootset. - // Set CONTAINER_TAG_SEEN while traversing. - for (auto container : rootset) { -#if TRACE_MEMORY - fprintf(stderr, "rootset %p\n", container); -#endif - phase3(container); - } -#if TRACE_GC_PHASES - dumpReachable("P3", state->toFree); -#endif - - // Traverse all elements, and collect those not having CONTAINER_TAG_SEEN and zero RC. - // Clear CONTAINER_TAG_SEEN while traversing on live elements, set in on dead elements. - for (auto container : *state->toFree) { - phase4(state, container); - } -#if TRACE_GC_PHASES - dumpReachable("P4", state->toFree); -#endif - - // Clear cycle candidates list. - state->toFree->clear(); - - processFinalizerQueue(state); state->gcInProgress = false; } @@ -949,7 +967,9 @@ void Kotlin_konan_internal_GC_stop(KRef) { if (memoryState->toFree != nullptr) { GarbageCollect(); konanDestructInstance(memoryState->toFree); + konanDestructInstance(memoryState->roots); memoryState->toFree = nullptr; + memoryState->roots = nullptr; } #endif } @@ -957,7 +977,8 @@ void Kotlin_konan_internal_GC_stop(KRef) { void Kotlin_konan_internal_GC_start(KRef) { #if USE_GC if (memoryState->toFree == nullptr) { - memoryState->toFree = konanConstructInstance(); + memoryState->toFree = konanConstructInstance(); + memoryState->roots = konanConstructInstance(); } #endif } @@ -980,14 +1001,14 @@ KInt Kotlin_konan_internal_GC_getThreshold(KRef) { KNativePtr CreateStablePointer(KRef any) { if (any == nullptr) return nullptr; - ::AddRef(any->container()); + AddRef(any->container()); return reinterpret_cast(any); } void DisposeStablePointer(KNativePtr pointer) { if (pointer == nullptr) return; KRef ref = reinterpret_cast(pointer); - ::Release(ref->container()); + Release(ref->container()); } OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { @@ -1009,6 +1030,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) { #if USE_GC if (root != nullptr) { auto state = memoryState; + auto container = root->container(); ContainerHeaderList todo; ContainerHeaderSet subgraph; @@ -1019,10 +1041,23 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) { if (subgraph.count(header) != 0) continue; subgraph.insert(header); - removeFreeable(state, header); - auto children = collectMutableReferred(header); - for (auto child : children) { - todo.push_back(child); +#if TRACE_MEMORY + fprintf(stderr, "Calling removeFreeable from ClearSubgraphReferences\n"); +#endif + traverseContainerReferredObjects(header, [&todo](ObjHeader* ref) { + auto child = ref->container(); + RuntimeAssert(!isArena(child), "A reference to local object is encountered"); + if (!isPermanent(child)) { + todo.push_back(child); + } + }); + } + for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) { + auto container = *it; + if (subgraph.find(container) != subgraph.end()) { + container->resetBuffered(); + container->setColor(CONTAINER_TAG_GC_BLACK); + *it = reinterpret_cast(reinterpret_cast(container) | 1); } } } diff --git a/runtime/src/main/cpp/Memory.h b/runtime/src/main/cpp/Memory.h index 704f2682e70..131907765cf 100644 --- a/runtime/src/main/cpp/Memory.h +++ b/runtime/src/main/cpp/Memory.h @@ -34,14 +34,12 @@ typedef enum { CONTAINER_TAG_PERMANENT = 2, // Stack objects, no need to free, children cleanup still shall be there. CONTAINER_TAG_STACK = 3, - // Container was seen during GC. - CONTAINER_TAG_SEEN = 4, // Shift to get actual counter. - CONTAINER_TAG_SHIFT = 3, + CONTAINER_TAG_SHIFT = 2, // Actual value to increment/decrement container by. Tag is in lower bits. CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT, - // Mask for container type, disregard seen bit. - CONTAINER_TAG_MASK = ((CONTAINER_TAG_INCREMENT >> 1) - 1), + // Mask for container type. + CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1, // Those bit masks are applied to objectCount_ field. // Shift to get actual object count. @@ -53,7 +51,8 @@ typedef enum { CONTAINER_TAG_GC_BLACK = 0, CONTAINER_TAG_GC_GRAY = 1, CONTAINER_TAG_GC_WHITE = 2, - CONTAINER_TAG_GC_PURPLE = 3 + CONTAINER_TAG_GC_PURPLE = 3, + CONTAINER_TAG_GC_BUFFERED = 4 } ContainerTag; typedef uint32_t container_offset_t; @@ -67,7 +66,6 @@ struct ContainerHeader { // Number of objects in the container. uint32_t objectCount_; - inline unsigned refCount() const { return refCount_ >> CONTAINER_TAG_SHIFT; } @@ -90,9 +88,18 @@ struct ContainerHeader { inline unsigned color() const { return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK; } - void setColor(unsigned color) { + inline void setColor(unsigned color) { objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color; } + inline bool buffered() const { + return (objectCount_ & CONTAINER_TAG_GC_BUFFERED) != 0; + } + inline void setBuffered() { + objectCount_ |= CONTAINER_TAG_GC_BUFFERED; + } + inline void resetBuffered() { + objectCount_ &= ~CONTAINER_TAG_GC_BUFFERED; + } }; struct ArrayHeader; @@ -170,48 +177,6 @@ inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) { return -obj->type_info()->instanceSize_ * obj->count_; } -// TODO: those two operations can be implemented by translator when storing -// reference to an object. -inline void AddRef(ContainerHeader* header) { - // Looking at container type we may want to skip AddRef() totally - // (non-escaping stack objects, constant objects). - switch (header->refCount_ & CONTAINER_TAG_MASK) { - case CONTAINER_TAG_STACK: - case CONTAINER_TAG_PERMANENT: - break; - case CONTAINER_TAG_NORMAL: - header->refCount_ += CONTAINER_TAG_INCREMENT; - break; - default: - RuntimeAssert(false, "unknown container type"); - break; - } -} - -void FreeContainer(ContainerHeader* header); - -// Release() returns 'true' iff container cannot be part of cycle (either NOCOUNT -// object or container was fully released and will be collected). -inline bool Release(ContainerHeader* header) { - switch (header->refCount_ & CONTAINER_TAG_MASK) { - case CONTAINER_TAG_PERMANENT: - case CONTAINER_TAG_STACK: - // permanent/stack containers aren't loop candidates. - return true; - case CONTAINER_TAG_NORMAL: - if ((header->refCount_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) { - FreeContainer(header); - return true; - } - break; - default: - RuntimeAssert(false, "unknown container type"); - break; - } - // Object with non-zero counter after release are loop candidates. - return false; -} - // Class representing arbitrary placement container. class Container { protected: @@ -224,20 +189,6 @@ class Container { obj->set_type_info(type_info); RuntimeAssert(obj->container() == header_, "Placement must match"); } - - public: - // Increment reference counter associated with container. - void AddRef() { - if (header_) ::AddRef(header_); - } - - // Decrement reference counter associated with container. - // For objects whith tricky lifetime (such as ones shared between threads objects) - // individual container per object (ObjectContainer) shall be created. - // As an alternative, such objects could be evacuated from short-lived containers. - void Release() { - if (header_) ::Release(header_); - } }; // Container for a single object.