azalea/kotlin-fork
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented Bacon's algorithm for cycle collection

Browse Source
This commit is contained in:
Igor Chevdar
2017-08-24 19:43:38 +05:00
parent 3522627ce3
commit 7e666f2d90
2 changed files with 361 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/src/main/cpp/Memory.cpp
+346 -311
View File
@@ -26,10 +26,8 @@
#include "Natives.h" #include "Natives.h"
// If garbage collection algorithm for cyclic garbage to be used. // If garbage collection algorithm for cyclic garbage to be used.
// We are using the Bacon's algorithm for GC (http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon03Pure.pdf).
#define USE_GC 1 #define USE_GC 1
// Optimize management of cyclic garbage (increases memory footprint).
// Not recommended for low-end embedded targets.
#define OPTIMIZE_GC 1
// Define to 1 to print all memory operations. // Define to 1 to print all memory operations.
#define TRACE_MEMORY 0 #define TRACE_MEMORY 0
// Trace garbage collection phases. // Trace garbage collection phases.
@@ -81,23 +79,27 @@ struct MemoryState {
// Finalizer queue. // Finalizer queue.
ContainerHeaderDeque* finalizerQueue; ContainerHeaderDeque* finalizerQueue;
// Set of references to release. /*
ContainerHeaderSet* toFree; * Typical scenario for GC is as following:
* we have 90% of objects with refcount = 0 which will be deleted during
* the first phase of the algorithm.
* We could mark them with a bit in order to tell the next two phases to skip them
* and thus requiring only one list, but the downside is that both of the
* next phases would iterate over the whole list of objects instead of only 10%.
*/
ContainerHeaderList* toFree; // List of all cycle candidates.
ContainerHeaderList* roots; // Real candidates excluding those with refcount = 0.
// How many GC suspend requests happened. // How many GC suspend requests happened.
int gcSuspendCount; int gcSuspendCount;
// How many candidate elements in toFree shall trigger collection. // How many candidate elements in toFree shall trigger collection.
size_t gcThreshold; size_t gcThreshold;
// If collection is in progress. // If collection is in progress.
bool gcInProgress; bool gcInProgress;
#if OPTIMIZE_GC #endif // USE_GC
// Cache backed by toFree set.
ContainerHeader** toFreeCache;
// Current number of elements in the cache.
uint32_t cacheSize;
#endif
#endif
}; };
void FreeContainer(ContainerHeader* header);
namespace { namespace {
// TODO: can we pass this variable as an explicit argument? // TODO: can we pass this variable as an explicit argument?
@@ -113,6 +115,10 @@ inline bool isPermanent(const ContainerHeader* header) {
return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT; return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_PERMANENT;
} }
inline bool isArena(const ContainerHeader* header) {
return (header->refCount_ & CONTAINER_TAG_MASK) == CONTAINER_TAG_STACK;
}
inline container_size_t alignUp(container_size_t size, int alignment) { inline container_size_t alignUp(container_size_t size, int alignment) {
return (size + alignment - 1) & ~(alignment - 1); return (size + alignment - 1) & ~(alignment - 1);
} }
@@ -145,14 +151,62 @@ inline bool isRefCounted(KConstRef object) {
return (object->container()->refCount_ & CONTAINER_TAG_MASK) == return (object->container()->refCount_ & CONTAINER_TAG_MASK) ==
CONTAINER_TAG_NORMAL; CONTAINER_TAG_NORMAL;
} }
} // namespace
extern "C" {
void objc_release(void* ptr);
}
inline void runDeallocationHooks(ObjHeader* obj) {
#if KONAN_OBJC_INTEROP
if (obj->type_info() == theObjCPointerHolderTypeInfo) {
void* objcPtr = *reinterpret_cast<void**>(obj + 1); // TODO: use more reliable layout description
objc_release(objcPtr);
}
#endif
}
inline void runDeallocationHooks(ContainerHeader* container) {
ObjHeader* obj = reinterpret_cast<ObjHeader*>(container + 1);
for (int index = 0; index < container->objectCount(); index++) {
runDeallocationHooks(obj);
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
}
}
static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) {
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(body) + typeInfo->objOffsets_[index]);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from DeinitInstanceBodyImpl\n");
#endif
UpdateRef(location, nullptr);
}
}
void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) {
DeinitInstanceBodyImpl(typeInfo, body);
}
namespace {
#if USE_GC #if USE_GC
inline void processFinalizerQueue(MemoryState* state) { inline void processFinalizerQueue(MemoryState* state) {
// TODO: reuse elements of finalizer queue for new allocations. // TODO: reuse elements of finalizer queue for new allocations.
while (!state->finalizerQueue->empty()) { while (!state->finalizerQueue->empty()) {
auto container = memoryState->finalizerQueue->back(); auto container = memoryState->finalizerQueue->back();
state->finalizerQueue->pop_back(); state->finalizerQueue->pop_back();
if ((reinterpret_cast<uintptr_t>(container) & 1) != 0) {
container = reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) & ~1);
#if TRACE_MEMORY
state->containers->erase(container);
#endif
runDeallocationHooks(container);
}
konanFreeMemory(container); konanFreeMemory(container);
state->allocCount--; state->allocCount--;
} }
@@ -174,135 +228,95 @@ inline void scheduleDestroyContainer(
} }
#if USE_GC #if !USE_GC
inline uint32_t hashOf(ContainerHeader* container) { inline void IncrementRC(ContainerHeader* container) {
uintptr_t value = reinterpret_cast<uintptr_t>(container); container->incRefCount();
return static_cast<uint32_t>(value >> 3) ^ static_cast<uint32_t>(static_cast<uint64_t>(value) >> 32);
} }
inline void DecrementRC(ContainerHeader* container) {
if (container->decRefCount() == 0) {
FreeContainer(container);
}
}
#else // USE_GC
inline uint32_t freeableSize(MemoryState* state) { inline uint32_t freeableSize(MemoryState* state) {
#if OPTIMIZE_GC
return state->cacheSize + state->toFree->size();
#else
return state->toFree->size(); return state->toFree->size();
#endif
} }
inline void addFreeable(MemoryState* state, ContainerHeader* container) { inline void IncrementRC(ContainerHeader* container) {
if (memoryState->toFree == nullptr || !isFreeable(container)) container->incRefCount();
return; container->setColor(CONTAINER_TAG_GC_BLACK);
#if OPTIMIZE_GC
auto hash = hashOf(container) % state->gcThreshold;
auto value = state->toFreeCache[hash];
if (value == container) {
return;
}
if (value == nullptr) {
memoryState->cacheSize++;
state->toFreeCache[hash] = container;
return;
}
state->toFree->insert(container);
if (value != (ContainerHeader*)0x1) {
memoryState->cacheSize--;
state->toFree->insert(value);
state->toFreeCache[hash] = (ContainerHeader*)0x1;
}
#else
state->toFree->insert(container);
#endif
if (state->gcSuspendCount == 0 &&
freeableSize(memoryState) > state->gcThreshold) {
GarbageCollect();
}
} }
inline void removeFreeable(MemoryState* state, ContainerHeader* container) { inline void DecrementRC(ContainerHeader* container) {
if (state->toFree == nullptr || !isFreeable(container)) if (container->decRefCount() == 0) {
return; FreeContainer(container);
#if OPTIMIZE_GC } else { // Possible root.
auto hash = hashOf(container) % state->gcThreshold; if (container->color() != CONTAINER_TAG_GC_PURPLE) {
auto value = state->toFreeCache[hash]; container->setColor(CONTAINER_TAG_GC_PURPLE);
if (value == container) { if (!container->buffered()) {
state->cacheSize--; container->setBuffered();
state->toFreeCache[hash] = nullptr; auto state = memoryState;
return; state->toFree->push_back(container);
} if (state->gcSuspendCount == 0 && freeableSize(state) > state->gcThreshold) {
#endif GarbageCollect();
state->toFree->erase(container); }
} }
// Must only be called in context of GC.
inline void flushFreeableCache(MemoryState* state) {
#if OPTIMIZE_GC
for (auto i = 0; i < state->gcThreshold; i++) {
if ((uintptr_t)state->toFreeCache[i] > 0x1) {
state->toFree->insert(state->toFreeCache[i]);
} }
} }
// Mass-clear cache.
memset(state->toFreeCache, 0,
sizeof(ContainerHeader*) * state->gcThreshold);
state->cacheSize = 0;
#endif
} }
inline void initThreshold(MemoryState* state, uint32_t gcThreshold) { inline void initThreshold(MemoryState* state, uint32_t gcThreshold) {
#if OPTIMIZE_GC
if (state->toFreeCache != nullptr) {
GarbageCollect();
konanFreeMemory(state->toFreeCache);
}
state->toFreeCache = reinterpret_cast<ContainerHeader**>(
konanAllocMemory(sizeof(ContainerHeader*) * gcThreshold));
state->cacheSize = 0;
#endif
state->gcThreshold = gcThreshold; state->gcThreshold = gcThreshold;
} }
#endif // USE_GC
// Must be vector or map 'container -> number', to keep reference counters correct. template<typename func>
ContainerHeaderList collectMutableReferred(ContainerHeader* header) { void traverseContainerObjectFields(ContainerHeader* container, func process) {
ContainerHeaderList result; ObjHeader* obj = reinterpret_cast<ObjHeader*>(container + 1);
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1); for (int object = 0; object < container->objectCount(); object++) {
for (int object = 0; object < header->objectCount(); object++) {
const TypeInfo* typeInfo = obj->type_info(); const TypeInfo* typeInfo = obj->type_info();
// TODO: generalize iteration over all references.
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) { for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>( ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]); reinterpret_cast<uintptr_t>(obj + 1) + typeInfo->objOffsets_[index]);
ObjHeader* ref = *location; process(location);
if (ref != nullptr && !isPermanent(ref->container())) {
result.push_back(ref->container());
}
} }
if (typeInfo == theArrayTypeInfo) { if (typeInfo == theArrayTypeInfo) {
ArrayHeader* array = obj->array(); ArrayHeader* array = obj->array();
for (int index = 0; index < array->count_; index++) { for (int index = 0; index < array->count_; index++) {
ObjHeader* ref = *ArrayAddressOfElementAt(array, index); process(ArrayAddressOfElementAt(array, index));
if (ref != nullptr && !isPermanent(ref->container())) {
result.push_back(ref->container());
}
} }
} }
obj = reinterpret_cast<ObjHeader*>( obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj)); reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
} }
return result;
} }
template<typename func>
void traverseContainerReferredObjects(ContainerHeader* container, func process) {
traverseContainerObjectFields(container, [process](ObjHeader** location) {
ObjHeader* ref = *location;
if (ref != nullptr) process(ref);
});
}
#if TRACE_MEMORY || USE_GC
void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) { void dumpWorker(const char* prefix, ContainerHeader* header, ContainerHeaderSet* seen) {
fprintf(stderr, "%s: %p (%08x): %d refs %s\n", fprintf(stderr, "%s: %p (%08x): %d refs\n",
prefix, prefix,
header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT, header, header->refCount_, header->refCount_ >> CONTAINER_TAG_SHIFT);
(header->refCount_ & CONTAINER_TAG_SEEN) != 0 ? "X" : "-");
seen->insert(header); seen->insert(header);
auto children = collectMutableReferred(header); traverseContainerReferredObjects(header, [prefix, seen](ObjHeader* ref) {
for (auto child : children) { auto child = ref->container();
if (seen->count(child) == 0) { RuntimeAssert(!isArena(child), "A reference to local object is encountered");
if (!isPermanent(child) && (seen->count(child) == 0)) {
dumpWorker(prefix, child, seen); dumpWorker(prefix, child, seen);
} }
} });
} }
void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) { void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
@@ -313,71 +327,159 @@ void dumpReachable(const char* prefix, const ContainerHeaderSet* roots) {
} }
} }
void phase1(ContainerHeader* header) { #endif
if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0)
return; void MarkRoots(MemoryState*);
header->refCount_ |= CONTAINER_TAG_SEEN; void DeleteCorpses(MemoryState*);
auto containers = collectMutableReferred(header); void ScanRoots(MemoryState*);
for (auto container : containers) { void CollectRoots(MemoryState*);
container->decRefCount(); void MarkGray(ContainerHeader* container);
phase1(container); void Scan(ContainerHeader* container);
void ScanBlack(ContainerHeader* container);
void CollectWhite(MemoryState*, ContainerHeader* container);
void CollectCycles(MemoryState* state) {
MarkRoots(state);
ScanRoots(state);
CollectRoots(state);
state->toFree->clear();
state->roots->clear();
}
void MarkRoots(MemoryState* state) {
for (auto container : *(state->toFree)) {
if ((reinterpret_cast<uintptr_t>(container) & 1) != 0)
continue;
auto color = container->color();
auto rcIsZero = container->refCount() == 0;
if (color == CONTAINER_TAG_GC_PURPLE && !rcIsZero) {
MarkGray(container);
state->roots->push_back(container);
} else {
container->resetBuffered();
if (color == CONTAINER_TAG_GC_BLACK && rcIsZero) {
scheduleDestroyContainer(state, reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1));
}
}
} }
} }
void phase2(ContainerHeader* header, ContainerHeaderSet* rootset) { void ScanRoots(MemoryState* state) {
if ((header->refCount_ & CONTAINER_TAG_SEEN) == 0) for (auto container : *(state->roots)) {
return; Scan(container);
if ((header->refCount_ >> CONTAINER_TAG_SHIFT) != 0)
rootset->insert(header);
header->refCount_ &= ~CONTAINER_TAG_SEEN;
auto containers = collectMutableReferred(header);
for (auto container : containers) {
phase2(container, rootset);
} }
} }
void phase3(ContainerHeader* header) { void CollectRoots(MemoryState* state) {
if ((header->refCount_ & CONTAINER_TAG_SEEN) != 0) { for (auto container : *(state->roots)) {
container->resetBuffered();
CollectWhite(state, container);
}
}
void MarkGray(ContainerHeader* container) {
if (container->color() == CONTAINER_TAG_GC_GRAY) return;
container->setColor(CONTAINER_TAG_GC_GRAY);
traverseContainerReferredObjects(container, [](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
childContainer->decRefCount();
MarkGray(childContainer);
}
});
}
void Scan(ContainerHeader* container) {
if (container->color() != CONTAINER_TAG_GC_GRAY) return;
if (container->refCount() != 0) {
ScanBlack(container);
return; return;
} }
header->refCount_ |= CONTAINER_TAG_SEEN; container->setColor(CONTAINER_TAG_GC_WHITE);
auto containers = collectMutableReferred(header); traverseContainerReferredObjects(container, [](ObjHeader* ref) {
for (auto container : containers) { auto childContainer = ref->container();
container->incRefCount(); RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
phase3(container); if (!isPermanent(childContainer)) {
Scan(childContainer);
}
});
}
void ScanBlack(ContainerHeader* container) {
container->setColor(CONTAINER_TAG_GC_BLACK);
traverseContainerReferredObjects(container, [](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
childContainer->incRefCount();
if (childContainer->color() != CONTAINER_TAG_GC_BLACK)
ScanBlack(childContainer);
}
});
}
void CollectWhite(MemoryState* state, ContainerHeader* container) {
if (container->color() != CONTAINER_TAG_GC_WHITE
|| container->buffered())
return;
container->setColor(CONTAINER_TAG_GC_BLACK);
traverseContainerReferredObjects(container, [state](ObjHeader* ref) {
auto childContainer = ref->container();
RuntimeAssert(!isArena(childContainer), "A reference to local object is encountered");
if (!isPermanent(childContainer)) {
CollectWhite(state, childContainer);
}
});
scheduleDestroyContainer(state, reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1));
}
inline void AddRef(ContainerHeader* header) {
// Looking at container type we may want to skip AddRef() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK:
case CONTAINER_TAG_PERMANENT:
break;
case CONTAINER_TAG_NORMAL:
IncrementRC(header);
break;
default:
RuntimeAssert(false, "unknown container type");
break;
} }
} }
void phase4(MemoryState* state, ContainerHeader* header) { inline void Release(ContainerHeader* header) {
auto refCount = header->refCount_ >> CONTAINER_TAG_SHIFT; // Looking at container type we may want to skip Release() totally
bool seen = (refCount > 0 && (header->refCount_ & CONTAINER_TAG_SEEN) == 0) || // (non-escaping stack objects, constant objects).
(refCount == 0 && (header->refCount_ & CONTAINER_TAG_SEEN) != 0); switch (header->refCount_ & CONTAINER_TAG_MASK) {
if (seen) return; case CONTAINER_TAG_PERMANENT:
case CONTAINER_TAG_STACK:
// Add to finalize queue and update seen bit. break;
if (refCount == 0) { case CONTAINER_TAG_NORMAL:
scheduleDestroyContainer(state, header); DecrementRC(header);
header->refCount_ |= CONTAINER_TAG_SEEN; break;
} else { default:
header->refCount_ &= ~CONTAINER_TAG_SEEN; RuntimeAssert(false, "unknown container type");
} break;
auto containers = collectMutableReferred(header);
for (auto container : containers) {
phase4(state, container);
} }
} }
#endif // USE_GC
// We use first slot as place to store frame-local arena container. // We use first slot as place to store frame-local arena container.
// TODO: create ArenaContainer object on the stack, so that we don't // TODO: create ArenaContainer object on the stack, so that we don't
// do two allocations per frame (ArenaContainer + actual container). // do two allocations per frame (ArenaContainer + actual container).
inline ArenaContainer* initedArena(ObjHeader** auxSlot) { inline ArenaContainer* initedArena(ObjHeader** auxSlot) {
ObjHeader* slotValue = *auxSlot; auto frame = asFrameOverlay(auxSlot);
if (slotValue) return reinterpret_cast<ArenaContainer*>(slotValue); #if TRACE_MEMORY
ArenaContainer* arena = konanConstructInstance<ArenaContainer>(); fprintf(stderr, "Initializing arena at %p\n", frame);
arena->Init(); #endif
*auxSlot = reinterpret_cast<ObjHeader*>(arena); auto arena = frame->arena;
if (!arena) {
arena = konanConstructInstance<ArenaContainer>();
arena->Init();
frame->arena = arena;
}
return arena; return arena;
} }
@@ -398,90 +500,41 @@ ContainerHeader* AllocContainer(size_t size) {
return result; return result;
} }
extern "C" {
void objc_release(void* ptr);
}
inline void runDeallocationHooks(ObjHeader* obj) {
#if KONAN_OBJC_INTEROP
if (obj->type_info() == theObjCPointerHolderTypeInfo) {
void* objcPtr = *reinterpret_cast<void**>(obj + 1); // TODO: use more reliable layout description
objc_release(objcPtr);
}
#endif
}
static inline void DeinitInstanceBodyImpl(const TypeInfo* typeInfo, void* body) {
for (int index = 0; index < typeInfo->objOffsetsCount_; index++) {
ObjHeader** location = reinterpret_cast<ObjHeader**>(
reinterpret_cast<uintptr_t>(body) + typeInfo->objOffsets_[index]);
UpdateRef(location, nullptr);
}
}
void DeinitInstanceBody(const TypeInfo* typeInfo, void* body) {
DeinitInstanceBodyImpl(typeInfo, body);
}
void FreeContainer(ContainerHeader* header) { void FreeContainer(ContainerHeader* header) {
RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed"); RuntimeAssert(!isPermanent(header), "this kind of container shalln't be freed");
auto state = memoryState; auto state = memoryState;
#if TRACE_MEMORY #if TRACE_MEMORY
if (isFreeable(header)) { if (isFreeable(header)) {
fprintf(stderr, "<<< free %p\n", header); fprintf(stderr, "<<< free<FreeContainer> %p\n", header);
state->containers->erase(header);
} }
#endif #endif
#if USE_GC
removeFreeable(state, header);
#endif
// Now let's clean all object's fields in this container. // Now let's clean all object's fields in this container.
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1); traverseContainerObjectFields(header, [](ObjHeader** location) {
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from FreeContainer\n");
#endif
for (int index = 0; index < header->objectCount(); index++) { UpdateRef(location, nullptr);
runDeallocationHooks(obj); });
const TypeInfo* typeInfo = obj->type_info();
DeinitInstanceBodyImpl(typeInfo, reinterpret_cast<void*>(obj + 1));
// Object arrays are *special*.
if (typeInfo == theArrayTypeInfo) {
ArrayHeader* array = obj->array();
ReleaseRefs(ArrayAddressOfElementAt(array, 0), array->count_);
}
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
}
// And release underlying memory. // And release underlying memory.
if (isFreeable(header)) { if (!isFreeable(header)) {
scheduleDestroyContainer(state, header); runDeallocationHooks(header);
} } else {
} header->setColor(CONTAINER_TAG_GC_BLACK);
if (!header->buffered()) {
runDeallocationHooks(header);
#if USE_GC
void FreeContainerNoRef(MemoryState* state, ContainerHeader* header) {
RuntimeAssert(isFreeable(header), "this kind of container shalln't be freed");
#if TRACE_MEMORY #if TRACE_MEMORY
fprintf(stderr, "<<< free %p\n", header); memoryState->containers->erase(header);
state->containers->erase(header);
#endif #endif
#if USE_GC
removeFreeable(state, header);
#endif
ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
for (int index = 0; index < header->objectCount(); index++) { scheduleDestroyContainer(state, header);
runDeallocationHooks(obj); }
obj = reinterpret_cast<ObjHeader*>(
reinterpret_cast<uintptr_t>(obj) + objectSize(obj));
} }
scheduleDestroyContainer(state, header);
} }
#endif
void ObjectContainer::Init(const TypeInfo* type_info) { void ObjectContainer::Init(const TypeInfo* type_info) {
RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object"); RuntimeAssert(type_info->instanceSize_ >= 0, "Must be an object");
@@ -525,9 +578,15 @@ void ArenaContainer::Init() {
} }
void ArenaContainer::Deinit() { void ArenaContainer::Deinit() {
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit start\n");
#endif
auto chunk = currentChunk_; auto chunk = currentChunk_;
while (chunk != nullptr) { while (chunk != nullptr) {
// FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set. // FreeContainer() doesn't release memory when CONTAINER_TAG_STACK is set.
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit free chunk\n");
#endif
FreeContainer(chunk->asHeader()); FreeContainer(chunk->asHeader());
chunk = chunk->next; chunk = chunk->next;
} }
@@ -537,7 +596,9 @@ void ArenaContainer::Deinit() {
chunk = chunk->next; chunk = chunk->next;
konanFreeMemory(toRemove); konanFreeMemory(toRemove);
} }
#if TRACE_MEMORY
fprintf(stderr, "Arena::Deinit end\n");
#endif
} }
bool ArenaContainer::allocContainer(container_size_t minSize) { bool ArenaContainer::allocContainer(container_size_t minSize) {
@@ -588,7 +649,7 @@ ObjHeader* ArenaContainer::PlaceObject(const TypeInfo* type_info) {
uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader); uint32_t size = type_info->instanceSize_ + sizeof(ObjHeader);
ObjHeader* result = reinterpret_cast<ObjHeader*>(place(size)); ObjHeader* result = reinterpret_cast<ObjHeader*>(place(size));
if (!result) { if (!result) {
return nullptr; return nullptr;
} }
currentChunk_->asHeader()->incObjectCount(); currentChunk_->asHeader()->incObjectCount();
setMeta(result, type_info); setMeta(result, type_info);
@@ -613,29 +674,13 @@ inline void AddRef(const ObjHeader* object) {
fprintf(stderr, "AddRef on %p in %p\n", object, object->container()); fprintf(stderr, "AddRef on %p in %p\n", object, object->container());
#endif #endif
AddRef(object->container()); AddRef(object->container());
#if USE_GC
// TODO: one could remove from toFree set here, as now container is reachable
// from the rootset, so cannot be cycle collection candidate.
// removeFreeable(memoryState, object->container());
#endif
} }
inline void ReleaseRef(const ObjHeader* object) { inline void ReleaseRef(const ObjHeader* object) {
#if TRACE_MEMORY #if TRACE_MEMORY
fprintf(stderr, "ReleaseRef on %p in %p\n", object, object->container()); fprintf(stderr, "ReleaseRef on %p in %p\n", object, object->container());
#endif #endif
#if USE_GC
// If object is not a cycle candidate - just return.
if (Release(object->container())) {
return;
}
#if TRACE_MEMORY
fprintf(stderr, "%p is release candidate\n", object->container());
#endif
addFreeable(memoryState, object->container());
#else // !USE_GC
Release(object->container()); Release(object->container());
#endif // USE_GC
} }
extern "C" { extern "C" {
@@ -649,6 +694,7 @@ MemoryState* InitMemory() {
== ==
offsetof(ObjHeader , container_offset_negative_), offsetof(ObjHeader , container_offset_negative_),
"Layout mismatch"); "Layout mismatch");
RuntimeAssert(sizeof(FrameOverlay) % sizeof(ObjHeader**) == 0, "Frame overlay should contain only pointers")
RuntimeAssert(memoryState == nullptr, "memory state must be clear"); RuntimeAssert(memoryState == nullptr, "memory state must be clear");
memoryState = konanConstructInstance<MemoryState>(); memoryState = konanConstructInstance<MemoryState>();
// TODO: initialize heap here. // TODO: initialize heap here.
@@ -659,7 +705,8 @@ MemoryState* InitMemory() {
#endif #endif
#if USE_GC #if USE_GC
memoryState->finalizerQueue = konanConstructInstance<ContainerHeaderDeque>(); memoryState->finalizerQueue = konanConstructInstance<ContainerHeaderDeque>();
memoryState->toFree = konanConstructInstance<ContainerHeaderSet>(); memoryState->toFree = konanConstructInstance<ContainerHeaderList>();
memoryState->roots = konanConstructInstance<ContainerHeaderList>();
memoryState->gcInProgress = false; memoryState->gcInProgress = false;
initThreshold(memoryState, kGcThreshold); initThreshold(memoryState, kGcThreshold);
memoryState->gcSuspendCount = 0; memoryState->gcSuspendCount = 0;
@@ -672,6 +719,9 @@ void DeinitMemory(MemoryState* memoryState) {
// Free all global objects, to ensure no memory leaks happens. // Free all global objects, to ensure no memory leaks happens.
for (auto location: *memoryState->globalObjects) { for (auto location: *memoryState->globalObjects) {
fprintf(stderr, "Release global in *%p: %p\n", location, *location); fprintf(stderr, "Release global in *%p: %p\n", location, *location);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from DeinitMemory\n");
#endif
UpdateRef(location, nullptr); UpdateRef(location, nullptr);
} }
konanDestructInstance(memoryState->globalObjects); konanDestructInstance(memoryState->globalObjects);
@@ -680,30 +730,26 @@ void DeinitMemory(MemoryState* memoryState) {
#if USE_GC #if USE_GC
GarbageCollect(); GarbageCollect();
RuntimeAssert(memoryState->toFree->size() == 0, "Some memory have not been released after GC");
konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->toFree);
memoryState->toFree = nullptr; konanDestructInstance(memoryState->roots);
#if OPTIMIZE_GC
if (memoryState->toFreeCache != nullptr) {
konanFreeMemory(memoryState->toFreeCache);
memoryState->toFreeCache = nullptr;
}
#endif
konanDestructInstance(memoryState->finalizerQueue); konanDestructInstance(memoryState->finalizerQueue);
memoryState->finalizerQueue = nullptr; memoryState->finalizerQueue = nullptr;
#endif // USE_GC #endif // USE_GC
#if TRACE_MEMORY
if (memoryState->allocCount > 0) { if (memoryState->allocCount > 0) {
fprintf(stderr, "*** Memory leaks, leaked %d containers ***\n", fprintf(stderr, "*** Memory leaks, leaked %d containers ***\n",
memoryState->allocCount); memoryState->allocCount);
#if TRACE_MEMORY
dumpReachable("", memoryState->containers); dumpReachable("", memoryState->containers);
konanDestructInstance(memoryState->containers);
memoryState->containers = nullptr;
#endif
} }
konanDestructInstance(memoryState->containers);
memoryState->containers = nullptr;
#else
RuntimeAssert(memoryState->allocCount == 0, "Memory leaks found");
#endif
konanFreeMemory(memoryState); konanFreeMemory(memoryState);
::memoryState = nullptr; ::memoryState = nullptr;
@@ -745,6 +791,9 @@ OBJ_GETTER(InitInstance,
} }
ObjHeader* object = AllocInstance(type_info, OBJ_RESULT); ObjHeader* object = AllocInstance(type_info, OBJ_RESULT);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance\n");
#endif
UpdateRef(location, object); UpdateRef(location, object);
#if KONAN_NO_EXCEPTIONS #if KONAN_NO_EXCEPTIONS
ctor(object); ctor(object);
@@ -760,7 +809,13 @@ OBJ_GETTER(InitInstance,
#endif #endif
return object; return object;
} catch (...) { } catch (...) {
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance #2\n");
#endif
UpdateRef(OBJ_RESULT, nullptr); UpdateRef(OBJ_RESULT, nullptr);
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from InitInstance #3\n");
#endif
UpdateRef(location, nullptr); UpdateRef(location, nullptr);
throw; throw;
} }
@@ -772,9 +827,7 @@ void SetRef(ObjHeader** location, const ObjHeader* object) {
fprintf(stderr, "SetRef *%p: %p\n", location, object); fprintf(stderr, "SetRef *%p: %p\n", location, object);
#endif #endif
*const_cast<const ObjHeader**>(location) = object; *const_cast<const ObjHeader**>(location) = object;
if (object != nullptr) { AddRef(object);
AddRef(object);
}
} }
ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) { ObjHeader** GetReturnSlotIfArena(ObjHeader** returnSlot, ObjHeader** localSlot) {
@@ -797,16 +850,20 @@ void UpdateReturnRef(ObjHeader** returnSlot, const ObjHeader* object) {
auto arena = initedArena(asArenaSlot(returnSlot)); auto arena = initedArena(asArenaSlot(returnSlot));
returnSlot = arena->getSlot(); returnSlot = arena->getSlot();
} }
#if TRACE_MEMORY
fprintf(stderr, "Calling UpdateRef from UpdateReturnRef\n");
#endif
UpdateRef(returnSlot, object); UpdateRef(returnSlot, object);
} }
void UpdateRef(ObjHeader** location, const ObjHeader* object) { void UpdateRef(ObjHeader** location, const ObjHeader* object) {
RuntimeAssert(!isArenaSlot(location), "must not be a slot"); RuntimeAssert(!isArenaSlot(location), "must not be a slot");
ObjHeader* old = *location; ObjHeader* old = *location;
if (old != object) {
#if TRACE_MEMORY #if TRACE_MEMORY
fprintf(stderr, "UpdateRef *%p: %p -> %p\n", location, old, object); fprintf(stderr, "UpdateRef *%p: %p -> %p\n", location, old, object);
fprintf(stderr, " *%p: %p -> %p\n", location, old == nullptr ? nullptr : old->container(), object == nullptr ? nullptr : object->container());
#endif #endif
if (old != object) {
if (object != nullptr) { if (object != nullptr) {
AddRef(object); AddRef(object);
} }
@@ -835,6 +892,9 @@ void LeaveFrame(ObjHeader** start, int count) {
#endif #endif
arena->Deinit(); arena->Deinit();
konanFreeMemory(arena); konanFreeMemory(arena);
#if TRACE_MEMORY
fprintf(stderr, "LeaveFrame: free arena done %p\n", arena);
#endif
} }
} }
@@ -843,6 +903,7 @@ void ReleaseRefs(ObjHeader** start, int count) {
fprintf(stderr, "ReleaseRefs %p .. %p\n", start, start + count); fprintf(stderr, "ReleaseRefs %p .. %p\n", start, start + count);
#endif #endif
ObjHeader** current = start; ObjHeader** current = start;
auto state = memoryState;
while (count-- > 0) { while (count-- > 0) {
ObjHeader* object = *current; ObjHeader* object = *current;
if (object != nullptr) { if (object != nullptr) {
@@ -855,64 +916,21 @@ void ReleaseRefs(ObjHeader** start, int count) {
} }
#if USE_GC #if USE_GC
void GarbageCollect() { void GarbageCollect() {
MemoryState* state = memoryState; MemoryState* state = memoryState;
RuntimeAssert(state->toFree != nullptr, "GC must not be stopped");
RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed"); RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed");
#if TRACE_MEMORY
fprintf(stderr, "Garbage collect\n");
#endif
state->gcInProgress = true; state->gcInProgress = true;
// Flush cache. while (state->toFree->size() > 0) {
flushFreeableCache(state); CollectCycles(state);
processFinalizerQueue(state);
// Traverse inner pointers in the closure of release candidates, and
// temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing.
#if TRACE_GC_PHASES
dumpReachable("P0", state->toFree);
#endif
for (auto container : *state->toFree) {
phase1(container);
} }
#if TRACE_GC_PHASES
dumpReachable("P1", state->toFree);
#endif
// Collect rootset from containers with non-zero reference counter. Those must
// be referenced from outside of newly released object graph.
// Clear CONTAINER_TAG_SEEN while traversing.
ContainerHeaderSet rootset;
for (auto container : *state->toFree) {
phase2(container, &rootset);
}
#if TRACE_GC_PHASES
dumpReachable("P2", state->toFree);
#endif
// Increment references for all elements reachable from the rootset.
// Set CONTAINER_TAG_SEEN while traversing.
for (auto container : rootset) {
#if TRACE_MEMORY
fprintf(stderr, "rootset %p\n", container);
#endif
phase3(container);
}
#if TRACE_GC_PHASES
dumpReachable("P3", state->toFree);
#endif
// Traverse all elements, and collect those not having CONTAINER_TAG_SEEN and zero RC.
// Clear CONTAINER_TAG_SEEN while traversing on live elements, set in on dead elements.
for (auto container : *state->toFree) {
phase4(state, container);
}
#if TRACE_GC_PHASES
dumpReachable("P4", state->toFree);
#endif
// Clear cycle candidates list.
state->toFree->clear();
processFinalizerQueue(state);
state->gcInProgress = false; state->gcInProgress = false;
} }
@@ -949,7 +967,9 @@ void Kotlin_konan_internal_GC_stop(KRef) {
if (memoryState->toFree != nullptr) { if (memoryState->toFree != nullptr) {
GarbageCollect(); GarbageCollect();
konanDestructInstance(memoryState->toFree); konanDestructInstance(memoryState->toFree);
konanDestructInstance(memoryState->roots);
memoryState->toFree = nullptr; memoryState->toFree = nullptr;
memoryState->roots = nullptr;
} }
#endif #endif
} }
@@ -957,7 +977,8 @@ void Kotlin_konan_internal_GC_stop(KRef) {
void Kotlin_konan_internal_GC_start(KRef) { void Kotlin_konan_internal_GC_start(KRef) {
#if USE_GC #if USE_GC
if (memoryState->toFree == nullptr) { if (memoryState->toFree == nullptr) {
memoryState->toFree = konanConstructInstance<ContainerHeaderSet>(); memoryState->toFree = konanConstructInstance<ContainerHeaderList>();
memoryState->roots = konanConstructInstance<ContainerHeaderList>();
} }
#endif #endif
} }
@@ -980,14 +1001,14 @@ KInt Kotlin_konan_internal_GC_getThreshold(KRef) {
KNativePtr CreateStablePointer(KRef any) { KNativePtr CreateStablePointer(KRef any) {
if (any == nullptr) return nullptr; if (any == nullptr) return nullptr;
::AddRef(any->container()); AddRef(any->container());
return reinterpret_cast<KNativePtr>(any); return reinterpret_cast<KNativePtr>(any);
} }
void DisposeStablePointer(KNativePtr pointer) { void DisposeStablePointer(KNativePtr pointer) {
if (pointer == nullptr) return; if (pointer == nullptr) return;
KRef ref = reinterpret_cast<KRef>(pointer); KRef ref = reinterpret_cast<KRef>(pointer);
::Release(ref->container()); Release(ref->container());
} }
OBJ_GETTER(DerefStablePointer, KNativePtr pointer) { OBJ_GETTER(DerefStablePointer, KNativePtr pointer) {
@@ -1009,6 +1030,7 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
#if USE_GC #if USE_GC
if (root != nullptr) { if (root != nullptr) {
auto state = memoryState; auto state = memoryState;
auto container = root->container(); auto container = root->container();
ContainerHeaderList todo; ContainerHeaderList todo;
ContainerHeaderSet subgraph; ContainerHeaderSet subgraph;
@@ -1019,10 +1041,23 @@ bool ClearSubgraphReferences(ObjHeader* root, bool checked) {
if (subgraph.count(header) != 0) if (subgraph.count(header) != 0)
continue; continue;
subgraph.insert(header); subgraph.insert(header);
removeFreeable(state, header); #if TRACE_MEMORY
auto children = collectMutableReferred(header); fprintf(stderr, "Calling removeFreeable from ClearSubgraphReferences\n");
for (auto child : children) { #endif
todo.push_back(child); traverseContainerReferredObjects(header, [&todo](ObjHeader* ref) {
auto child = ref->container();
RuntimeAssert(!isArena(child), "A reference to local object is encountered");
if (!isPermanent(child)) {
todo.push_back(child);
}
});
}
for (auto it = state->toFree->begin(); it != state->toFree->end(); ++it) {
auto container = *it;
if (subgraph.find(container) != subgraph.end()) {
container->resetBuffered();
container->setColor(CONTAINER_TAG_GC_BLACK);
*it = reinterpret_cast<ContainerHeader*>(reinterpret_cast<uintptr_t>(container) | 1);
} }
} }
} }
runtime/src/main/cpp/Memory.h
+15 -64
View File
@@ -34,14 +34,12 @@ typedef enum {
CONTAINER_TAG_PERMANENT = 2, CONTAINER_TAG_PERMANENT = 2,
// Stack objects, no need to free, children cleanup still shall be there. // Stack objects, no need to free, children cleanup still shall be there.
CONTAINER_TAG_STACK = 3, CONTAINER_TAG_STACK = 3,
// Container was seen during GC.
CONTAINER_TAG_SEEN = 4,
// Shift to get actual counter. // Shift to get actual counter.
CONTAINER_TAG_SHIFT = 3, CONTAINER_TAG_SHIFT = 2,
// Actual value to increment/decrement container by. Tag is in lower bits. // Actual value to increment/decrement container by. Tag is in lower bits.
CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT, CONTAINER_TAG_INCREMENT = 1 << CONTAINER_TAG_SHIFT,
// Mask for container type, disregard seen bit. // Mask for container type.
CONTAINER_TAG_MASK = ((CONTAINER_TAG_INCREMENT >> 1) - 1), CONTAINER_TAG_MASK = CONTAINER_TAG_INCREMENT - 1,
// Those bit masks are applied to objectCount_ field. // Those bit masks are applied to objectCount_ field.
// Shift to get actual object count. // Shift to get actual object count.
@@ -53,7 +51,8 @@ typedef enum {
CONTAINER_TAG_GC_BLACK = 0, CONTAINER_TAG_GC_BLACK = 0,
CONTAINER_TAG_GC_GRAY = 1, CONTAINER_TAG_GC_GRAY = 1,
CONTAINER_TAG_GC_WHITE = 2, CONTAINER_TAG_GC_WHITE = 2,
CONTAINER_TAG_GC_PURPLE = 3 CONTAINER_TAG_GC_PURPLE = 3,
CONTAINER_TAG_GC_BUFFERED = 4
} ContainerTag; } ContainerTag;
typedef uint32_t container_offset_t; typedef uint32_t container_offset_t;
@@ -67,7 +66,6 @@ struct ContainerHeader {
// Number of objects in the container. // Number of objects in the container.
uint32_t objectCount_; uint32_t objectCount_;
inline unsigned refCount() const { inline unsigned refCount() const {
return refCount_ >> CONTAINER_TAG_SHIFT; return refCount_ >> CONTAINER_TAG_SHIFT;
} }
@@ -90,9 +88,18 @@ struct ContainerHeader {
inline unsigned color() const { inline unsigned color() const {
return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK; return objectCount_ & CONTAINER_TAG_GC_COLOR_MASK;
} }
void setColor(unsigned color) { inline void setColor(unsigned color) {
objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color; objectCount_ = (objectCount_ & ~CONTAINER_TAG_GC_COLOR_MASK) | color;
} }
inline bool buffered() const {
return (objectCount_ & CONTAINER_TAG_GC_BUFFERED) != 0;
}
inline void setBuffered() {
objectCount_ |= CONTAINER_TAG_GC_BUFFERED;
}
inline void resetBuffered() {
objectCount_ &= ~CONTAINER_TAG_GC_BUFFERED;
}
}; };
struct ArrayHeader; struct ArrayHeader;
@@ -170,48 +177,6 @@ inline uint32_t ArrayDataSizeBytes(const ArrayHeader* obj) {
return -obj->type_info()->instanceSize_ * obj->count_; return -obj->type_info()->instanceSize_ * obj->count_;
} }
// TODO: those two operations can be implemented by translator when storing
// reference to an object.
inline void AddRef(ContainerHeader* header) {
// Looking at container type we may want to skip AddRef() totally
// (non-escaping stack objects, constant objects).
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_STACK:
case CONTAINER_TAG_PERMANENT:
break;
case CONTAINER_TAG_NORMAL:
header->refCount_ += CONTAINER_TAG_INCREMENT;
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
}
void FreeContainer(ContainerHeader* header);
// Release() returns 'true' iff container cannot be part of cycle (either NOCOUNT
// object or container was fully released and will be collected).
inline bool Release(ContainerHeader* header) {
switch (header->refCount_ & CONTAINER_TAG_MASK) {
case CONTAINER_TAG_PERMANENT:
case CONTAINER_TAG_STACK:
// permanent/stack containers aren't loop candidates.
return true;
case CONTAINER_TAG_NORMAL:
if ((header->refCount_ -= CONTAINER_TAG_INCREMENT) == CONTAINER_TAG_NORMAL) {
FreeContainer(header);
return true;
}
break;
default:
RuntimeAssert(false, "unknown container type");
break;
}
// Object with non-zero counter after release are loop candidates.
return false;
}
// Class representing arbitrary placement container. // Class representing arbitrary placement container.
class Container { class Container {
protected: protected:
@@ -224,20 +189,6 @@ class Container {
obj->set_type_info(type_info); obj->set_type_info(type_info);
RuntimeAssert(obj->container() == header_, "Placement must match"); RuntimeAssert(obj->container() == header_, "Placement must match");
} }
public:
// Increment reference counter associated with container.
void AddRef() {
if (header_) ::AddRef(header_);
}
// Decrement reference counter associated with container.
// For objects whith tricky lifetime (such as ones shared between threads objects)
// individual container per object (ObjectContainer) shall be created.
// As an alternative, such objects could be evacuated from short-lived containers.
void Release() {
if (header_) ::Release(header_);
}
}; };
// Container for a single object. // Container for a single object.