azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented Bacon's algorithm for cycle collection

Browse Source
This commit is contained in:
Igor Chevdar
2017-08-24 19:43:38 +05:00
parent 3522627ce3
commit 7e666f2d90
2 changed files with 361 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/src/main/cpp/Memory.cpp
+346 -311
View File
@@ -26,10 +26,8 @@
#include "Natives.h"
// If garbage collection algorithm for cyclic garbage to be used.
// We are using the Bacon's algorithm for GC (http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon03Pure.pdf).
#define USE_GC 1
// Optimize management of cyclic garbage (increases memory footprint).
// Not recommended for low-end embedded targets.
#define OPTIMIZE_GC 1
// Define to 1 to print all memory operations.
#define TRACE_MEMORY 0
// Trace garbage collection phases.
@@ -81,23 +79,27 @@ struct MemoryState {
// Finalizer queue.
ContainerHeaderDeque* finalizerQueue;
// Set of references to release.
ContainerHeaderSet* toFree;
/*
* Typical scenario for GC is as following:
* we have 90% of objects with refcount = 0 which will be deleted during
* the first phase of the algorithm.
* We could mark them with a bit in order to tell the next two phases to skip them
* and thus requiring only one list, but the downside is that both of the
* next phases would iterate over the whole list of objects instead of only 10%.
*/
ContainerHeaderList* toFree; // List of all cycle candidates.
ContainerHeaderList* roots; // Real candidates excluding those with refcount = 0.
// How many GC suspend requests happened.
int gcSuspendCount;
// How many candidate elements in toFree shall trigger collection.
size_t gcThreshold;
// If collection is in progress.
bool gcInProgress;
#if OPTIMIZE_GC
// Cache backed by toFree set.
ContainerHeader** toFreeCache;
// Current number of elements in the cache.
uint32_t cacheSize;
#endif
#endif
#endif // USE_GC
};
void FreeContainer(ContainerHeader* header);
namespace {
// TODO: can we pass this variable as an explicit argument?
@@ -113,6 +115,10 @@ inline bool isPermanent(const ContainerHeader* header) {
return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
}
inline bool isArena(const ContainerHeader* header) {
return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK;
}
inline container_size_t alignUp(container_size_t size, int alignment) {
return (size + alignment - 1) & ~(alignment - 1);
}
@@ -145,14 +151,62 @@ inline bool isRefCounted(KConstRef object) {
return (object->container()->refCount_ & CONTAINER_TAG_MASK) ==
CONTAINER_TAG_NORMAL;
}
} // namespace
extern "C" {
void objc_release(void* ptr);
}
inline void runDeallocationHooks(ObjHeader* obj) {
#if KONAN_OBJC_INTEROP
if (obj->type_info() == theObjCPointerHolderTypeInfo) {
void* objcPtr = *reinterpret_cast<void**>(obj + 1); // TODO: use more reliable layout description
objc_release(objcPtr);
}
#endif
}
inline void runDeallocationHooks(ContainerHeader* container) {
ObjHeader* obj = reinterpret_cast<ObjHeader*>(container + 1);
for (int index = 0; index < container->objectCount(); index++) {
runDeallocationHooks(obj);
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
}
}
static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) {
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(body) + typeInfo->objOffsets_[index]);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from DeinitInstanceBodyImpl\n");
#endif
UpdateRef(location, nullptr);
}
}
void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) {
DeinitInstanceBodyImpl(typeInfo, body);
}
namespace {
#if USE_GC
inline void processFinalizerQueue(MemoryState* state) {
// TODO: reuse elements of finalizer queue for new allocations.
while (!state->finalizerQueue->empty()) {
auto container = memoryState->finalizerQueue->back();
state->finalizerQueue->pop_back();
if ((reinterpret_cast<uintptr_t>(container) & 1) != 0) {
container = reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) & ~1);
#if TRACE_MEMORY
state->containers->erase(container);
#endif
runDeallocationHooks(container);
}
konanFreeMemory(container);
state->allocCount--;
}
@@ -174,135 +228,95 @@ inline void scheduleDestroyContainer(
}
#if USE_GC
#if !USE_GC
inline uint32_t hashOf(ContainerHeader* container) {
uintptr_t value = reinterpret_cast<uintptr_t>(container);
return static_cast<uint32_t>(value >> 3) ^ static_cast<uint32_t>(static_cast<uint64_t>(value) >> 32);
inline void IncrementRC(ContainerHeader* container) {
container->incRefCount();
}
inline void DecrementRC(ContainerHeader* container) {
if (container->decRefCount() == 0) {
FreeContainer(container);
}
}
#else // USE_GC
inline uint32_t freeableSize(MemoryState* state) {
#if OPTIMIZE_GC
return state->cacheSize + state->toFree->size();
#else
return state->toFree->size();
#endif
}
inline void addFreeable(MemoryState* state, ContainerHeader* container) {
if (memoryState->toFree == nullptr || !isFreeable(container))
return;
#if OPTIMIZE_GC
auto hash = hashOf(container) % state->gcThreshold;
auto value = state->toFreeCache[hash];
if (value == container) {
return;
}
if (value == nullptr) {
memoryState->cacheSize++;
state->toFreeCache[hash] = container;
return;
}
state->toFree->insert(container);
if (value != (ContainerHeader*)0x1) {
memoryState->cacheSize--;
state->toFree->insert(value);
state->toFreeCache[hash] = (ContainerHeader*)0x1;
}
#else
state->toFree->insert(container);
#endif
if (state->gcSuspendCount == 0 &&
freeableSize(memoryState) > state->gcThreshold) {
GarbageCollect();
}
inline void IncrementRC(ContainerHeader* container) {
container->incRefCount();
container->setColor(CONTAINER_TAG_GC_BLACK);
}
inline void removeFreeable(MemoryState* state, ContainerHeader* container) {
if (state->toFree == nullptr || !isFreeable(container))
return;
#if OPTIMIZE_GC
auto hash = hashOf(container) % state->gcThreshold;
auto value = state->toFreeCache[hash];
if (value == container) {
state->cacheSize--;
state->toFreeCache[hash] = nullptr;
return;
}
#endif
state->toFree->erase(container);
}
// Must only be called in context of GC.
inline void flushFreeableCache(MemoryState* state) {
#if OPTIMIZE_GC
for (auto i = 0; i < state->gcThreshold; i++) {
if ((uintptr_t)state->toFreeCache[i] > 0x1) {
state->toFree->insert(state->toFreeCache[i]);
inline void DecrementRC(ContainerHeader* container) {
if (container->decRefCount() == 0) {
FreeContainer(container);
} else { // Possible root.
if (container->color() != CONTAINER_TAG_GC_PURPLE) {
container->setColor(CONTAINER_TAG_GC_PURPLE);
if (!container->buffered()) {
container->setBuffered();
auto state = memoryState;
state->toFree->push_back(container);
if (state->gcSuspendCount == 0 && freeableSize(state) > state->gcThreshold) {
GarbageCollect();
}
}
}
}
// Mass-clear cache.
memset(state->toFreeCache, 0,
sizeof(ContainerHeader*) * state->gcThreshold);
state->cacheSize = 0;
#endif
}
inline void initThreshold(MemoryState* state, uint32_t gcThreshold) {
#if OPTIMIZE_GC
if (state->toFreeCache != nullptr) {
GarbageCollect();
konanFreeMemory(state->toFreeCache);
}
state->toFreeCache = reinterpret_cast<ContainerHeader**>(
konanAllocMemory(sizeof(ContainerHeader*) * gcThreshold));
state->cacheSize = 0;
#endif
state->gcThreshold = gcThreshold;
}
#endif // USE_GC
// Must be vector or map 'container -> number', to keep reference counters correct.
ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
ContainerHeaderList result;
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
for (int object = 0; object < header->objectCount(); object++) {
template<typename func>
void traverseContainerObjectFields(ContainerHeader* container, func process) {
ObjHeader* obj = reinterpret_cast<ObjHeader*>(container + 1);
for (int object = 0; object < container->objectCount(); object++) {
const TypeInfo* typeInfo = obj->type_info();
// TODO: generalize iteration over all references.
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
ObjHeader* ref = *location;
if (ref != nullptr && !isPermanent(ref->container())) {
result.push_back(ref->container());
}
process(location);
}
if (typeInfo == theArrayTypeInfo) {
ArrayHeader* array = obj->array();
for (int index = 0; index < array->count_; index++) {
ObjHeader* ref = *ArrayAddressOfElementAt(array, index);
if (ref != nullptr && !isPermanent(ref->container())) {
result.push_back(ref->container());
}
process(ArrayAddressOfElementAt(array, index));
}
}
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
}
return result;
}
template<typename func>
void traverseContainerReferredObjects(ContainerHeader* container, func process) {
traverseContainerObjectFields(container, [process](ObjHeader** location) {
ObjHeader* ref = *location;
if (ref != nullptr) process(ref);
});
}
#if TRACE_MEMORY || USE_GC
void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) {
fprintf(stderr, "%s: %p (%08x): %d refs %s\n",
fprintf(stderr, "%s: %p (%08x): %d refs\n",
prefix,
header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT,
(header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT);
seen->insert(header);
auto children = collectMutableReferred(header);
for (auto child : children) {
if (seen->count(child) == 0) {
traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) {
auto child = ref->container();
RuntimeAssert(!isArena(child), "A reference to local object is encountered");
if (!isPermanent(child) && (seen->count(child) == 0)) {
dumpWorker(prefix, child, seen);
}
}
});
}
void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
@@ -313,71 +327,159 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
}
}
void phase1(ContainerHeader* header) {
if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0)
return;
header->refCount_ |= CONTAINER_TAG_SEEN;
auto containers = collectMutableReferred(header);
for (auto container : containers) {
container->decRefCount();
phase1(container);
#endif
void MarkRoots(MemoryState*);
void DeleteCorpses(MemoryState*);
void ScanRoots(MemoryState*);
void CollectRoots(MemoryState*);
void MarkGray(ContainerHeader* container);
void Scan(ContainerHeader* container);
void ScanBlack(ContainerHeader* container);
void CollectWhite(MemoryState*, ContainerHeader* container);
void CollectCycles(MemoryState* state) {
MarkRoots(state);
ScanRoots(state);
CollectRoots(state);
state->toFree->clear();
state->roots->clear();
}
void MarkRoots(MemoryState* state) {
for (auto container : *(state->toFree)) {
if ((reinterpret_cast<uintptr_t>(container) & 1) != 0)
continue;
auto color = container->color();
auto rcIsZero = container->refCount() == 0;
if (color == CONTAINER_TAG_GC_PURPLE && !rcIsZero) {
MarkGray(container);
state->roots->push_back(container);
} else {
container->resetBuffered();
if (color == CONTAINER_TAG_GC_BLACK && rcIsZero) {
scheduleDestroyContainer(state, reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1));
}
}
}
}
void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) {
if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0)
return;
if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0)
rootset->insert(header);
header->refCount_ &= ~CONTAINER_TAG_SEEN;
auto containers = collectMutableReferred(header);
for (auto container : containers) {
phase2(container, rootset);
void ScanRoots(MemoryState* state) {
for (auto container : *(state->roots)) {
Scan(container);
}
}
void phase3(ContainerHeader* header) {
if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) {
void CollectRoots(MemoryState* state) {
for (auto container : *(state->roots)) {
container->resetBuffered();
CollectWhite(state, container);
}
}
void MarkGray(ContainerHeader* container) {
if (container->color() == CONTAINER_TAG_GC_GRAY) return;
container->setColor(CONTAINER_TAG_GC_GRAY);
traverseContainerReferredObjects(container, [](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
childContainer->decRefCount();
MarkGray(childContainer);
}
});
}
void Scan(ContainerHeader* container) {
if (container->color() != CONTAINER_TAG_GC_GRAY) return;
if (container->refCount() != 0) {
ScanBlack(container);
return;
}
header->refCount_ |= CONTAINER_TAG_SEEN;
auto containers = collectMutableReferred(header);
for (auto container : containers) {
container->incRefCount();
phase3(container);
container->setColor(CONTAINER_TAG_GC_WHITE);
traverseContainerReferredObjects(container, [](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
Scan(childContainer);
}
});
}
void ScanBlack(ContainerHeader* container) {
container->setColor(CONTAINER_TAG_GC_BLACK);
traverseContainerReferredObjects(container, [](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
childContainer->incRefCount();
if (childContainer->color() != CONTAINER_TAG_GC_BLACK)
ScanBlack(childContainer);
}
});
}
void CollectWhite(MemoryState* state, ContainerHeader* container) {
if (container->color() != CONTAINER_TAG_GC_WHITE
|| container->buffered())
return;
container->setColor(CONTAINER_TAG_GC_BLACK);
traverseContainerReferredObjects(container, [state](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
CollectWhite(state, childContainer);
}
});
scheduleDestroyContainer(state, reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1));
}
inline void AddRef(ContainerHeader* header) {
// Looking at container type we may want to skip AddRef() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK:
case CONTAINER_TAG_PERMANENT:
break;
case CONTAINER_TAG_NORMAL:
IncrementRC(header);
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
}
void phase4(MemoryState* state, ContainerHeader* header) {
auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT;
bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) ||
(refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0);
if (seen) return;
// Add to finalize queue and update seen bit.
if (refCount == 0) {
scheduleDestroyContainer(state, header);
header->refCount_ |= CONTAINER_TAG_SEEN;
} else {
header->refCount_ &= ~CONTAINER_TAG_SEEN;
}
auto containers = collectMutableReferred(header);
for (auto container : containers) {
phase4(state, container);
inline void Release(ContainerHeader* header) {
// Looking at container type we may want to skip Release() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_PERMANENT:
case CONTAINER_TAG_STACK:
break;
case CONTAINER_TAG_NORMAL:
DecrementRC(header);
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
}
#endif // USE_GC
// We use first slot as place to store frame-local arena container.
// TODO: create ArenaContainer object on the stack, so that we don't
// do two allocations per frame (ArenaContainer + actual container).
inline ArenaContainer* initedArena(ObjHeader** auxSlot) {
ObjHeader* slotValue = *auxSlot;
if (slotValue) return reinterpret_cast<ArenaContainer*>(slotValue);
ArenaContainer* arena = konanConstructInstance<ArenaContainer>();
arena->Init();
*auxSlot = reinterpret_cast<ObjHeader*>(arena);
auto frame = asFrameOverlay(auxSlot);
#if TRACE_MEMORY
fprintf(stderr, "Initializing arena at %p\n", frame);
#endif
auto arena = frame->arena;
if (!arena) {
arena = konanConstructInstance<ArenaContainer>();
arena->Init();
frame->arena = arena;
}
return arena;
}
@@ -398,90 +500,41 @@ ContainerHeader* AllocContainer(size_t size) {
return result;
}
extern "C" {
void objc_release(void* ptr);
}
inline void runDeallocationHooks(ObjHeader* obj) {
#if KONAN_OBJC_INTEROP
if (obj->type_info() == theObjCPointerHolderTypeInfo) {
void* objcPtr = *reinterpret_cast<void**>(obj + 1); // TODO: use more reliable layout description
objc_release(objcPtr);
}
#endif
}
static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) {
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(body) + typeInfo->objOffsets_[index]);
UpdateRef(location, nullptr);
}
}
void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) {
DeinitInstanceBodyImpl(typeInfo, body);
}
void FreeContainer(ContainerHeader* header) {
RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
auto state = memoryState;
#if TRACE_MEMORY
if (isFreeable(header)) {
fprintf(stderr, "<<< free %p\n", header);
state->containers->erase(header);
fprintf(stderr, "<<< free<FreeContainer> %p\n", header);
}
#endif
#if USE_GC
removeFreeable(state, header);
#endif
// Now let's clean all object's fields in this container.
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
traverseContainerObjectFields(header, [](ObjHeader** location) {
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from FreeContainer\n");
#endif
for (int index = 0; index < header->objectCount(); index++) {
runDeallocationHooks(obj);
const TypeInfo* typeInfo = obj->type_info();
DeinitInstanceBodyImpl(typeInfo, reinterpret_cast<void*>(obj + 1));
// Object arrays are *special*.
if (typeInfo == theArrayTypeInfo) {
ArrayHeader* array = obj->array();
ReleaseRefs(ArrayAddressOfElementAt(array, 0), array->count_);
}
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
}
UpdateRef(location, nullptr);
});
// And release underlying memory.
if (isFreeable(header)) {
scheduleDestroyContainer(state, header);
}
}
if (!isFreeable(header)) {
runDeallocationHooks(header);
} else {
header->setColor(CONTAINER_TAG_GC_BLACK);
if (!header->buffered()) {
runDeallocationHooks(header);
#if USE_GC
void FreeContainerNoRef(MemoryState* state, ContainerHeader* header) {
RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed");
#if TRACE_MEMORY
fprintf(stderr, "<<< free %p\n", header);
state->containers->erase(header);
memoryState->containers->erase(header);
#endif
#if USE_GC
removeFreeable(state, header);
#endif
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
for (int index = 0; index < header->objectCount(); index++) {
runDeallocationHooks(obj);
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
scheduleDestroyContainer(state, header);
}
}
scheduleDestroyContainer(state, header);
}
#endif
void ObjectContainer::Init(const TypeInfo* type_info) {
RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
@@ -525,9 +578,15 @@ void ArenaContainer::Init() {
}
void ArenaContainer::Deinit() {
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit start\n");
#endif
auto chunk = currentChunk_;
while (chunk != nullptr) {
// FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set.
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit free chunk\n");
#endif
FreeContainer(chunk->asHeader());
chunk = chunk->next;
}
@@ -537,7 +596,9 @@ void ArenaContainer::Deinit() {
chunk = chunk->next;
konanFreeMemory(toRemove);
}
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit end\n");
#endif
}
bool ArenaContainer::allocContainer(container_size_t minSize) {
@@ -588,7 +649,7 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
ObjHeader* result = reinterpret_cast<ObjHeader*>(place(size));
if (!result) {
return nullptr;
return nullptr;
}
currentChunk_->asHeader()->incObjectCount();
setMeta(result, type_info);
@@ -613,29 +674,13 @@ inline void AddRef(const ObjHeader* object) {
fprintf(stderr, "AddRef on %p in %p\n", object, object->container());
#endif
AddRef(object->container());
#if USE_GC
// TODO: one could remove from toFree set here, as now container is reachable
// from the rootset, so cannot be cycle collection candidate.
// removeFreeable(memoryState, object->container());
#endif
}
inline void ReleaseRef(const ObjHeader* object) {
#if TRACE_MEMORY
fprintf(stderr, "ReleaseRef on %p in %p\n", object, object->container());
#endif
#if USE_GC
// If object is not a cycle candidate - just return.
if (Release(object->container())) {
return;
}
#if TRACE_MEMORY
fprintf(stderr, "%p is release candidate\n", object->container());
#endif
addFreeable(memoryState, object->container());
#else // !USE_GC
Release(object->container());
#endif // USE_GC
}
extern "C" {
@@ -649,6 +694,7 @@ MemoryState* InitMemory() {
==
offsetof(ObjHeader , container_offset_negative_),
"Layout mismatch");
RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers")
RuntimeAssert(memoryState == nullptr, "memory state must be clear");
memoryState = konanConstructInstance<MemoryState>();
// TODO: initialize heap here.
@@ -659,7 +705,8 @@ MemoryState* InitMemory() {
#endif
#if USE_GC
memoryState->finalizerQueue = konanConstructInstance<ContainerHeaderDeque>();
memoryState->toFree = konanConstructInstance<ContainerHeaderSet>();
memoryState->toFree = konanConstructInstance<ContainerHeaderList>();
memoryState->roots = konanConstructInstance<ContainerHeaderList>();
memoryState->gcInProgress = false;
initThreshold(memoryState, kGcThreshold);
memoryState->gcSuspendCount = 0;
@@ -672,6 +719,9 @@ void DeinitMemory(MemoryState* memoryState) {
// Free all global objects, to ensure no memory leaks happens.
for (auto location: *memoryState->globalObjects) {
fprintf(stderr, "Release global in *%p: %p\n", location, *location);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from DeinitMemory\n");
#endif
UpdateRef(location, nullptr);
}
konanDestructInstance(memoryState->globalObjects);
@@ -680,30 +730,26 @@ void DeinitMemory(MemoryState* memoryState) {
#if USE_GC
GarbageCollect();
RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC");
konanDestructInstance(memoryState->toFree);
memoryState->toFree = nullptr;
#if OPTIMIZE_GC
if (memoryState->toFreeCache != nullptr) {
konanFreeMemory(memoryState->toFreeCache);
memoryState->toFreeCache = nullptr;
}
#endif
konanDestructInstance(memoryState->roots);
konanDestructInstance(memoryState->finalizerQueue);
memoryState->finalizerQueue = nullptr;
#endif // USE_GC
#if TRACE_MEMORY
if (memoryState->allocCount > 0) {
fprintf(stderr, "*** Memory leaks, leaked %d containers ***\n",
memoryState->allocCount);
#if TRACE_MEMORY
dumpReachable("", memoryState->containers);
konanDestructInstance(memoryState->containers);
memoryState->containers = nullptr;
#endif
}
konanDestructInstance(memoryState->containers);
memoryState->containers = nullptr;
#else
RuntimeAssert(memoryState->allocCount == 0, "Memory leaks found");
#endif
konanFreeMemory(memoryState);
::memoryState = nullptr;
@@ -745,6 +791,9 @@ OBJ_GETTER(InitInstance,
}
ObjHeader* object = AllocInstance(type_info, OBJ_RESULT);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance\n");
#endif
UpdateRef(location, object);
#if KONAN_NO_EXCEPTIONS
ctor(object);
@@ -760,7 +809,13 @@ OBJ_GETTER(InitInstance,
#endif
return object;
} catch (...) {
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance #2\n");
#endif
UpdateRef(OBJ_RESULT, nullptr);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance #3\n");
#endif
UpdateRef(location, nullptr);
throw;
}
@@ -772,9 +827,7 @@ void SetRef(ObjHeader** location, const ObjHeader* object) {
fprintf(stderr, "SetRef *%p: %p\n", location, object);
#endif
*const_cast<const ObjHeader**>(location) = object;
if (object != nullptr) {
AddRef(object);
}
AddRef(object);
}
ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) {
@@ -797,16 +850,20 @@ void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
auto arena = initedArena(asArenaSlot(returnSlot));
returnSlot = arena->getSlot();
}
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from UpdateReturnRef\n");
#endif
UpdateRef(returnSlot, object);
}
void UpdateRef(ObjHeader** location, const ObjHeader* object) {
RuntimeAssert(!isArenaSlot(location), "must not be a slot");
ObjHeader* old = *location;
if (old != object) {
#if TRACE_MEMORY
fprintf(stderr, "UpdateRef *%p: %p -> %p\n", location, old, object);
fprintf(stderr, " *%p: %p -> %p\n", location, old == nullptr ? nullptr : old->container(), object == nullptr ? nullptr : object->container());
#endif
if (old != object) {
if (object != nullptr) {
AddRef(object);
}
@@ -835,6 +892,9 @@ void LeaveFrame(ObjHeader** start, int count) {
#endif
arena->Deinit();
konanFreeMemory(arena);
#if TRACE_MEMORY
fprintf(stderr, "LeaveFrame: free arena done %p\n", arena);
#endif
}
}
@@ -843,6 +903,7 @@ void ReleaseRefs(ObjHeader** start, int count) {
fprintf(stderr, "ReleaseRefs %p .. %p\n", start, start + count);
#endif
ObjHeader** current = start;
auto state = memoryState;
while (count-- > 0) {
ObjHeader* object = *current;
if (object != nullptr) {
@@ -855,64 +916,21 @@ void ReleaseRefs(ObjHeader** start, int count) {
}
#if USE_GC
void GarbageCollect() {
MemoryState* state = memoryState;
RuntimeAssert(state->toFree != nullptr, "GC must not be stopped");
RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed");
#if TRACE_MEMORY
fprintf(stderr, "Garbage collect\n");
#endif
state->gcInProgress = true;
// Flush cache.
flushFreeableCache(state);
// Traverse inner pointers in the closure of release candidates, and
// temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing.
#if TRACE_GC_PHASES
dumpReachable("P0", state->toFree);
#endif
for (auto container : *state->toFree) {
phase1(container);
while (state->toFree->size() > 0) {
CollectCycles(state);
processFinalizerQueue(state);
}
#if TRACE_GC_PHASES
dumpReachable("P1", state->toFree);
#endif
// Collect rootset from containers with non-zero reference counter. Those must
// be referenced from outside of newly released object graph.
// Clear CONTAINER_TAG_SEEN while traversing.
ContainerHeaderSet rootset;
for (auto container : *state->toFree) {
phase2(container, &rootset);
}
#if TRACE_GC_PHASES
dumpReachable("P2", state->toFree);
#endif
// Increment references for all elements reachable from the rootset.
// Set CONTAINER_TAG_SEEN while traversing.
for (auto container : rootset) {
#if TRACE_MEMORY
fprintf(stderr, "rootset %p\n", container);
#endif
phase3(container);
}
#if TRACE_GC_PHASES
dumpReachable("P3", state->toFree);
#endif
// Traverse all elements, and collect those not having CONTAINER_TAG_SEEN and zero RC.
// Clear CONTAINER_TAG_SEEN while traversing on live elements, set in on dead elements.
for (auto container : *state->toFree) {
phase4(state, container);
}
#if TRACE_GC_PHASES
dumpReachable("P4", state->toFree);
#endif
// Clear cycle candidates list.
state->toFree->clear();
processFinalizerQueue(state);
state->gcInProgress = false;
}
@@ -949,7 +967,9 @@ void Kotlin_konan_internal_GC_stop(KRef) {
if (memoryState->toFree != nullptr) {
GarbageCollect();
konanDestructInstance(memoryState->toFree);
konanDestructInstance(memoryState->roots);
memoryState->toFree = nullptr;
memoryState->roots = nullptr;
}
#endif
}
@@ -957,7 +977,8 @@ void Kotlin_konan_internal_GC_stop(KRef) {
void Kotlin_konan_internal_GC_start(KRef) {
#if USE_GC
if (memoryState->toFree == nullptr) {
memoryState->toFree = konanConstructInstance<ContainerHeaderSet>();
memoryState->toFree = konanConstructInstance<ContainerHeaderList>();
memoryState->roots = konanConstructInstance<ContainerHeaderList>();
}
#endif
}
@@ -980,14 +1001,14 @@ KInt Kotlin_konan_internal_GC_getThreshold(KRef) {
KNativePtr CreateStablePointer(KRef any) {
if (any == nullptr) return nullptr;
::AddRef(any->container());
AddRef(any->container());
return reinterpret_cast<KNativePtr>(any);
}
void DisposeStablePointer(KNativePtr pointer) {
if (pointer == nullptr) return;
KRef ref = reinterpret_cast<KRef>(pointer);
::Release(ref->container());
Release(ref->container());
}
OBJ_GETTER(DerefStablePointer, KNativePtr pointer) {
@@ -1009,6 +1030,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
#if USE_GC
if (root != nullptr) {
auto state = memoryState;
auto container = root->container();
ContainerHeaderList todo;
ContainerHeaderSet subgraph;
@@ -1019,10 +1041,23 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
if (subgraph.count(header) != 0)
continue;
subgraph.insert(header);
removeFreeable(state, header);
auto children = collectMutableReferred(header);
for (auto child : children) {
todo.push_back(child);
#if TRACE_MEMORY
fprintf(stderr, "Calling removeFreeable from ClearSubgraphReferences\n");
#endif
traverseContainerReferredObjects(header, [&todo](ObjHeader* ref) {
auto child = ref->container();
RuntimeAssert(!isArena(child), "A reference to local object is encountered");
if (!isPermanent(child)) {
todo.push_back(child);
}
});
}
for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) {
auto container = *it;
if (subgraph.find(container) != subgraph.end()) {
container->resetBuffered();
container->setColor(CONTAINER_TAG_GC_BLACK);
*it = reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1);
}
}
}
runtime/src/main/cpp/Memory.h
+15 -64
View File
@@ -34,14 +34,12 @@ typedef enum {
CONTAINER_TAG_PERMANENT = 2,
// Stack objects, no need to free, children cleanup still shall be there.
CONTAINER_TAG_STACK = 3,
// Container was seen during GC.
CONTAINER_TAG_SEEN = 4,
// Shift to get actual counter.
CONTAINER_TAG_SHIFT = 3,
CONTAINER_TAG_SHIFT = 2,
// Actual value to increment/decrement container by. Tag is in lower bits.
CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
// Mask for container type, disregard seen bit.
CONTAINER_TAG_MASK = ((CONTAINER_TAG_INCREMENT >> 1) - 1),
// Mask for container type.
CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
// Those bit masks are applied to objectCount_ field.
// Shift to get actual object count.
@@ -53,7 +51,8 @@ typedef enum {
CONTAINER_TAG_GC_BLACK = 0,
CONTAINER_TAG_GC_GRAY = 1,
CONTAINER_TAG_GC_WHITE = 2,
CONTAINER_TAG_GC_PURPLE = 3
CONTAINER_TAG_GC_PURPLE = 3,
CONTAINER_TAG_GC_BUFFERED = 4
} ContainerTag;
typedef uint32_t container_offset_t;
@@ -67,7 +66,6 @@ struct ContainerHeader {
// Number of objects in the container.
uint32_t objectCount_;
inline unsigned refCount() const {
return refCount_ >> CONTAINER_TAG_SHIFT;
}
@@ -90,9 +88,18 @@ struct ContainerHeader {
inline unsigned color() const {
return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK;
}
void setColor(unsigned color) {
inline void setColor(unsigned color) {
objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
}
inline bool buffered() const {
return (objectCount_ & CONTAINER_TAG_GC_BUFFERED) != 0;
}
inline void setBuffered() {
objectCount_ |= CONTAINER_TAG_GC_BUFFERED;
}
inline void resetBuffered() {
objectCount_ &= ~CONTAINER_TAG_GC_BUFFERED;
}
};
struct ArrayHeader;
@@ -170,48 +177,6 @@ inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
return -obj->type_info()->instanceSize_ * obj->count_;
}
// TODO: those two operations can be implemented by translator when storing
// reference to an object.
inline void AddRef(ContainerHeader* header) {
// Looking at container type we may want to skip AddRef() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK:
case CONTAINER_TAG_PERMANENT:
break;
case CONTAINER_TAG_NORMAL:
header->refCount_ += CONTAINER_TAG_INCREMENT;
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
}
void FreeContainer(ContainerHeader* header);
// Release() returns 'true' iff container cannot be part of cycle (either NOCOUNT
// object or container was fully released and will be collected).
inline bool Release(ContainerHeader* header) {
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_PERMANENT:
case CONTAINER_TAG_STACK:
// permanent/stack containers aren't loop candidates.
return true;
case CONTAINER_TAG_NORMAL:
if ((header->refCount_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) {
FreeContainer(header);
return true;
}
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
// Object with non-zero counter after release are loop candidates.
return false;
}
// Class representing arbitrary placement container.
class Container {
protected:
@@ -224,20 +189,6 @@ class Container {
obj->set_type_info(type_info);
RuntimeAssert(obj->container() == header_, "Placement must match");
}
public:
// Increment reference counter associated with container.
void AddRef() {
if (header_) ::AddRef(header_);
}
// Decrement reference counter associated with container.
// For objects whith tricky lifetime (such as ones shared between threads objects)
// individual container per object (ObjectContainer) shall be created.
// As an alternative, such objects could be evacuated from short-lived containers.
void Release() {
if (header_) ::Release(header_);
}
};
// Container for a single object.