[K/N] Estimate allocator overhead

This increases the boundary for the next GC scheduling by how much is
estimated is currently being wasted by the allocator due to paging. This
is necessary, since the gc scheduling boundary is set based on the size
of the live set at the last GC, while the current memory use is only
tracked in whole allocator pages. This patch avoids some of the
situations where the GC would be triggered again immediately upon
completion, which happens with many sparsely filled pages.

kotlin-native/runtime/src/alloc/common/cpp/Allocator.hpp

@@ -51,6 +51,8 @@ public:
     // TODO: Move into AllocatorTestSupport.hpp
     void clearForTests() noexcept;
 
+    size_t estimateOverheadPerThread() noexcept;
+
 private:
     std::unique_ptr<Impl> impl_;
 };

kotlin-native/runtime/src/alloc/custom/cpp/AllocatorImpl.cpp

@@ -6,6 +6,7 @@
 #include "AllocatorImpl.hpp"
 
 #include "GCApi.hpp"
+#include "Heap.hpp"
 
 using namespace kotlin;
 
@@ -50,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept {
     impl_->heap().ClearForTests();
 }
 
+size_t alloc::Allocator::estimateOverheadPerThread() noexcept {
+    return impl_->heap().EstimateOverheadPerThread();
+}
+
 void alloc::initObjectPool() noexcept {}
 
 void alloc::compactObjectPoolInCurrentThread() noexcept {}

kotlin-native/runtime/src/alloc/custom/cpp/Heap.cpp

@@ -116,6 +116,16 @@ std::vector<ObjHeader*> Heap::GetAllocatedObjects() noexcept {
     return unfinalized;
 }
 
+size_t Heap::EstimateOverheadPerThread() noexcept {
+    size_t overHeadPerThread = 0;
+    if (!nextFitPages_.isEmpty()) overHeadPerThread += NEXT_FIT_PAGE_SIZE;
+    for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
+        if (!fixedBlockPages_[blockSize].isEmpty()) overHeadPerThread += FIXED_BLOCK_PAGE_SIZE;
+    }
+    if (!extraObjectPages_.isEmpty()) overHeadPerThread += EXTRA_OBJECT_PAGE_SIZE;
+    return  overHeadPerThread;
+}
+
 void Heap::ClearForTests() noexcept {
     for (int blockSize = 0; blockSize <= FIXED_BLOCK_PAGE_MAX_BLOCK_SIZE; ++blockSize) {
         fixedBlockPages_[blockSize].ClearForTests();

kotlin-native/runtime/src/alloc/custom/cpp/Heap.hpp

@@ -39,6 +39,7 @@ public:
 
     void AddToFinalizerQueue(FinalizerQueue queue) noexcept;
     FinalizerQueue ExtractFinalizerQueue() noexcept;
+    size_t EstimateOverheadPerThread() noexcept;
 
     // Test method
     std::vector<ObjHeader*> GetAllocatedObjects() noexcept;

kotlin-native/runtime/src/alloc/custom/cpp/PageStore.hpp

@@ -78,6 +78,13 @@ public:
         return page;
     }
 
+    bool isEmpty() noexcept {
+        return empty_.isEmpty() &&
+            ready_.isEmpty() &&
+            used_.isEmpty() &&
+            unswept_.isEmpty();
+    }
+
     ~PageStore() noexcept {
         T* page;
         while ((page = empty_.Pop())) page->Destroy();

kotlin-native/runtime/src/alloc/legacy/cpp/AllocatorImpl.cpp

@@ -51,6 +51,10 @@ void alloc::Allocator::clearForTests() noexcept {
     impl_->objectFactory().ClearForTests();
 }
 
+size_t alloc::Allocator::estimateOverheadPerThread() noexcept {
+    return 0;
+}
+
 gc::GC::ObjectData& alloc::objectDataForObject(ObjHeader* object) noexcept {
     return ObjectFactoryImpl::NodeRef::From(object).ObjectData();
 }

kotlin-native/runtime/src/gc/cms/cpp/ConcurrentMarkAndSweep.cpp

@@ -176,8 +176,10 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     // By this point all the alive heap must be marked.
     // All the mutations (incl. allocations) after this method will be subject for the next GC.
     // This should really be done by each individual thread while waiting
+    int threadCount = 0;
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
         thread.allocator().prepareForGC();
+        ++threadCount;
     }
     allocator_.prepareForGC();
 
@@ -206,7 +208,7 @@ void gc::ConcurrentMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     }
     finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
-    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes());
+    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread());
     state_.finish(epoch);
     gcHandle.finalizersScheduled(finalizerQueue.size());
     gcHandle.finished();

kotlin-native/runtime/src/gc/stms/cpp/SameThreadMarkAndSweep.cpp

@@ -71,8 +71,10 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     gc::processWeaks<DefaultProcessWeaksTraits>(gcHandle, mm::SpecialRefRegistry::instance());
 
     // This should really be done by each individual thread while waiting
+    int threadCount = 0;
     for (auto& thread : kotlin::mm::ThreadRegistry::Instance().LockForIter()) {
         thread.allocator().prepareForGC();
+        ++threadCount;
     }
     allocator_.prepareForGC();
 
@@ -96,7 +98,7 @@ void gc::SameThreadMarkAndSweep::PerformFullGC(int64_t epoch) noexcept {
     finalizerQueue.TransferAllFrom(allocator_.impl().heap().ExtractFinalizerQueue());
 #endif
 
-    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes());
+    scheduler.onGCFinish(epoch, gcHandle.getKeptSizeBytes() + threadCount * allocator_.estimateOverheadPerThread());
 
     resumeTheWorld(gcHandle);
 

kotlin-native/runtime/src/gcScheduler/adaptive/cpp/GCSchedulerImpl.hpp

@@ -64,6 +64,7 @@ public:
 
     void onGCStart() noexcept {
         regularIntervalPacer_.OnPerformFullGC();
+        heapGrowthController_.unsetBoundaries();
         timer_.restart(config_.regularGcInterval());
     }
 

kotlin-native/runtime/src/gcScheduler/common/cpp/HeapGrowthController.hpp

@@ -31,6 +31,10 @@ public:
         triggerHeapBytes_ = targetHeapBytes_ * config_.heapTriggerCoefficient.load(std::memory_order_relaxed);
     }
 
+    void unsetBoundaries() noexcept {
+        targetHeapBytes_ = triggerHeapBytes_ = std::numeric_limits<int64_t>::max();
+    }
+
     // Can be called by any thread.
     MemoryBoundary boundaryForHeapSize(size_t totalAllocatedBytes) noexcept {
         if (totalAllocatedBytes >= targetHeapBytes_) {