Optimize memory management (#480)

runtime/src/main/cpp/Memory.cpp

@@ -29,6 +29,9 @@
 
 // If garbage collection algorithm for cyclic garbage to be used.
 #define USE_GC 1
+// Optmize management of cyclic garbage (increases memory footprint).
+// Not recommended for low-end embedded targets.
+#define OPTIMIZE_GC 1
 // Define to 1 to print all memory operations.
 #define TRACE_MEMORY 0
 // Trace garbage collection phases.
@@ -49,8 +52,8 @@ constexpr container_size_t kObjectAlignment = 8;
 
 #if USE_GC
 // Collection threshold default (collect after having so many elements in the
-// release candidates set).
+// release candidates set). Better be a prime number.
-constexpr size_t kGcThreshold = 10000;
+constexpr size_t kGcThreshold = 9341;
 #endif
 
 #if TRACE_MEMORY || USE_GC
@@ -79,12 +82,18 @@ struct MemoryState {
   size_t gcThreshold;
   // If collection is in progress.
   bool gcInProgress;
+#if OPTIMIZE_GC
+  // Cache backed by toFree set.
+  ContainerHeader** toFreeCache;
+  // Current number of elements in the cache.
+  uint32_t cacheSize;
+#endif
 #endif
 };
 
 namespace {
 
-// TODO: remove this global.
+// TODO: can we pass this variable as an explicit argument?
 __thread MemoryState* memoryState = nullptr;
 
 // TODO: use those allocators for STL containers as well.
@@ -120,6 +129,91 @@ inline ObjHeader** asArenaSlot(ObjHeader** slot) {
 
 #if USE_GC
 
+inline uint32_t hashOf(ContainerHeader* container) {
+  uintptr_t value = reinterpret_cast<uintptr_t>(container);
+  return static_cast<uint32_t>(value >> 3) ^ static_cast<uint32_t>(value >> 32);
+}
+
+inline uint32_t freeableSize(MemoryState* state) {
+#if OPTIMIZE_GC
+  return state->cacheSize + state->toFree->size();
+#else
+  return state->toFree->size();
+#endif
+}
+
+inline void addFreeable(MemoryState* state, ContainerHeader* container) {
+  if (memoryState->toFree == nullptr || !isFreeable(container))
+    return;
+#if OPTIMIZE_GC
+  auto hash = hashOf(container) % state->gcThreshold;
+  auto value = state->toFreeCache[hash];
+  if (value == container) {
+    return;
+  }
+  if (value == nullptr) {
+    memoryState->cacheSize++;
+    state->toFreeCache[hash] = container;
+    return;
+  }
+  state->toFree->insert(container);
+  if (value != (ContainerHeader*)0x1) {
+    memoryState->cacheSize--;
+    state->toFree->insert(value);
+    state->toFreeCache[hash] = (ContainerHeader*)0x1;
+  }
+#else
+  state->toFree->insert(container);
+#endif
+  if (state->gcSuspendCount == 0 &&
+      freeableSize(memoryState) > state->gcThreshold) {
+    GarbageCollect();
+  }
+}
+
+inline void removeFreeable(MemoryState* state, ContainerHeader* container) {
+  if (state->toFree == nullptr || !isFreeable(container))
+    return;
+#if OPTIMIZE_GC
+  auto hash = hashOf(container) % state->gcThreshold;
+  auto value = state->toFreeCache[hash];
+  if (value == container) {
+    state->cacheSize--;
+    state->toFreeCache[hash] = nullptr;
+    return;
+  }
+#endif
+  state->toFree->erase(container);
+}
+
+// Must only be called in context of GC.
+inline void flushFreeableCache(MemoryState* state) {
+#if OPTIMIZE_GC
+  for (auto i = 0; i < state->gcThreshold; i++) {
+    if ((uintptr_t)state->toFreeCache[i] > 0x1) {
+      state->toFree->insert(state->toFreeCache[i]);
+    }
+  }
+  // Mass-clear cache.
+  memset(state->toFreeCache, 0,
+         sizeof(ContainerHeader*) * state->gcThreshold);
+  state->cacheSize = 0;
+#endif
+}
+
+inline void initThreshold(MemoryState* state, uint32_t gcThreshold) {
+#if OPTIMIZE_GC
+  if (state->toFreeCache != nullptr) {
+    GarbageCollect();
+    freeMemory(state->toFreeCache);
+  }
+  state->toFreeCache = allocMemory<ContainerHeader*>(
+      sizeof(ContainerHeader*) * gcThreshold);
+  state->cacheSize = 0;
+#endif
+  state->gcThreshold = gcThreshold;
+}
+
 // Must be vector or map 'container -> number', to keep reference counters correct.
 ContainerHeaderList collectMutableReferred(ContainerHeader* header) {
   ContainerHeaderList result;
@@ -272,8 +366,7 @@ void FreeContainer(ContainerHeader* header) {
 #endif
 
 #if USE_GC
-  if (memoryState->toFree && isFreeable(header))
+  removeFreeable(memoryState, header);
-    memoryState->toFree->erase(header);
 #endif
   // Now let's clean all object's fields in this container.
   ObjHeader* obj = reinterpret_cast<ObjHeader*>(header + 1);
@@ -310,8 +403,7 @@ void FreeContainerNoRef(ContainerHeader* header) {
   memoryState->containers->erase(header);
 #endif
 #if USE_GC
-  if (memoryState->toFree)
+  removeFreeable(memoryState, header);
-    memoryState->toFree->erase(header);
 #endif
   memoryState->allocCount--;
   freeMemory(header);
@@ -433,7 +525,7 @@ inline void AddRef(const ObjHeader* object) {
 #if USE_GC
   // TODO: one could remove from toFree set here, as now container is reachable
   // from the rootset, so cannot be cycle collection candidate.
-  // memoryState->toFree->erase(object->container());
+  // removeFreeable(memoryState, object->container());
 #endif
 }
 
@@ -449,12 +541,7 @@ inline void ReleaseRef(const ObjHeader* object) {
 #if TRACE_MEMORY
   fprintf(stderr, "%p is release candidate\n", object->container());
 #endif
-  if (memoryState->toFree != nullptr) {
+  addFreeable(memoryState, object->container());
-    memoryState->toFree->insert(object->container());
-    if (memoryState->gcSuspendCount == 0 &&
-        memoryState->toFree->size() > memoryState->gcThreshold)
-      GarbageCollect();
-  }
 #else // !USE_GC
   Release(object->container());
 #endif // USE_GC
@@ -472,7 +559,7 @@ MemoryState* InitMemory() {
                 offsetof(ObjHeader  , container_offset_negative_),
                 "Layout mismatch");
   RuntimeAssert(memoryState == nullptr, "memory state must be clear");
-  memoryState = new MemoryState();
+  memoryState = allocMemory<MemoryState>(sizeof(MemoryState));
   // TODO: initialize heap here.
   memoryState->allocCount = 0;
 #if TRACE_MEMORY
@@ -482,7 +569,7 @@ MemoryState* InitMemory() {
 #if USE_GC
   memoryState->toFree = new ContainerHeaderSet();
   memoryState->gcInProgress = false;
-  memoryState->gcThreshold = kGcThreshold;
+  initThreshold(memoryState, kGcThreshold);
   memoryState->gcSuspendCount = 0;
 #endif
   return memoryState;
@@ -514,7 +601,7 @@ void DeinitMemory(MemoryState* memoryState) {
 #endif
   }
 
-  delete memoryState;
+  freeMemory(memoryState);
   ::memoryState = nullptr;
 }
 
@@ -645,30 +732,35 @@ void ReleaseRefs(ObjHeader** start, int count) {
 
 #if USE_GC
 void GarbageCollect() {
-  RuntimeAssert(memoryState->toFree != nullptr, "GC must not be stopped");
+  MemoryState* state = memoryState;
-  RuntimeAssert(!memoryState->gcInProgress, "Recursive GC is disallowed");
+  RuntimeAssert(state->toFree != nullptr, "GC must not be stopped");
-  memoryState->gcInProgress = true;
+  RuntimeAssert(!state->gcInProgress, "Recursive GC is disallowed");
+
+  // Flush cache.
+  flushFreeableCache(state);
+
+  state->gcInProgress = true;
   // Traverse inner pointers in the closure of release candidates, and
   // temporary decrement refs on them. Set CONTAINER_TAG_SEEN while traversing.
 #if TRACE_GC_PHASES
-  dumpReachable("P0", memoryState->toFree);
+  dumpReachable("P0", state->toFree);
 #endif
-  for (auto container : *memoryState->toFree) {
+  for (auto container : *state->toFree) {
     phase1(container);
   }
 #if TRACE_GC_PHASES
-  dumpReachable("P1", memoryState->toFree);
+  dumpReachable("P1", state->toFree);
 #endif
 
   // Collect rootset from containers with non-zero reference counter. Those must
   // be referenced from outside of newly released object graph.
   // Clear CONTAINER_TAG_SEEN while traversing.
   ContainerHeaderSet rootset;
-  for (auto container : *memoryState->toFree) {
+  for (auto container : *state->toFree) {
     phase2(container, &rootset);
   }
 #if TRACE_GC_PHASES
-  dumpReachable("P2", memoryState->toFree);
+  dumpReachable("P2", state->toFree);
 #endif
 
   // Increment references for all elements reachable from the rootset.
@@ -680,28 +772,28 @@ void GarbageCollect() {
     phase3(container);
   }
 #if TRACE_GC_PHASES
-  dumpReachable("P3", memoryState->toFree);
+  dumpReachable("P3", state->toFree);
 #endif
 
   // Traverse all elements, and collect those not having CONTAINER_TAG_SEEN and zero RC.
   // Clear CONTAINER_TAG_SEEN while traversing on live elements, set in on dead elements.
   ContainerHeaderSet toRemove;
-  for (auto container : *memoryState->toFree) {
+  for (auto container : *state->toFree) {
     phase4(container, &toRemove);
   }
 #if TRACE_GC_PHASES
-  dumpReachable("P4", memoryState->toFree);
+  dumpReachable("P4", state->toFree);
 #endif
 
   // Clear cycle candidates list.
-  memoryState->toFree->clear();
+  state->toFree->clear();
 
   for (auto header : toRemove) {
     RuntimeAssert((header->refCount_ & CONTAINER_TAG_SEEN) != 0, "Must be not seen");
     FreeContainerNoRef(header);
   }
 
-  memoryState->gcInProgress = false;
+  state->gcInProgress = false;
 }
 
 #endif // USE_GC
@@ -720,10 +812,11 @@ void Kotlin_konan_internal_GC_suspend(KRef) {
 
 void Kotlin_konan_internal_GC_resume(KRef) {
 #if USE_GC
-  if (memoryState->gcSuspendCount > 0) {
+  MemoryState* state = memoryState;
-    memoryState->gcSuspendCount--;
+  if (state->gcSuspendCount > 0) {
-    if (memoryState->toFree != nullptr &&
+    state->gcSuspendCount--;
-        memoryState->toFree->size() >= memoryState->gcThreshold) {
+    if (state->toFree != nullptr &&
+        freeableSize(state) >= state->gcThreshold) {
       GarbageCollect();
     }
   }
@@ -751,7 +844,7 @@ void Kotlin_konan_internal_GC_start(KRef) {
 void Kotlin_konan_internal_GC_setThreshold(KRef, KInt value) {
 #if USE_GC
   if (value > 0) {
-    memoryState->gcThreshold = value;
+    initThreshold(memoryState, value);
   }
 #endif
 }